You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by sh...@apache.org on 2016/11/01 16:12:24 UTC
[1/6] airavata git commit: Merge pull request #1 from apache/develop
Repository: airavata
Updated Branches:
refs/heads/develop 325d1630c -> 6d51651ca
Merge pull request #1 from apache/develop
Merge airvata develop with local develop
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/7dd85164
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/7dd85164
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/7dd85164
Branch: refs/heads/develop
Commit: 7dd85164103ea492351e25695b4c9b48a9840703
Parents: 59142dd 76c9eaf
Author: Gourav Shenoy <sh...@gmail.com>
Authored: Sun Oct 23 13:01:27 2016 -0400
Committer: GitHub <no...@github.com>
Committed: Sun Oct 23 13:01:27 2016 -0400
----------------------------------------------------------------------
airavata-api/airavata-api-server/pom.xml | 17 +-
.../server/handler/AiravataServerHandler.java | 1073 +-
.../java/org/apache/airavata/api/Airavata.java | 74743 +++++++++++------
.../main/resources/lib/airavata/Airavata.cpp | 42654 ++++++----
.../src/main/resources/lib/airavata/Airavata.h | 2838 +-
.../lib/airavata/Airavata_server.skeleton.cpp | 371 +-
.../resources/lib/airavata/airavata_api_types.h | 1 +
.../airavata/compute_resource_model_types.cpp | 28 +-
.../lib/airavata/compute_resource_model_types.h | 13 +-
.../resources/lib/airavata/job_model_types.cpp | 30 +-
.../resources/lib/airavata/job_model_types.h | 12 +-
.../lib/airavata/process_model_types.cpp | 60 +-
.../lib/airavata/process_model_types.h | 22 +-
.../resources/lib/airavata/task_model_types.cpp | 60 +-
.../resources/lib/airavata/task_model_types.h | 18 +-
.../user_resource_profile_model_constants.cpp | 34 +
.../user_resource_profile_model_constants.h | 41 +
.../user_resource_profile_model_types.cpp | 737 +
.../user_resource_profile_model_types.h | 326 +
.../resources/lib/Airavata/API/Airavata.php | 18554 ++--
.../Model/AppCatalog/ComputeResource/Types.php | 2 +
.../lib/Airavata/Model/Process/Types.php | 44 +-
.../resources/lib/Airavata/Model/Task/Types.php | 44 +-
.../resources/lib/Airavata/Model/job/Types.php | 22 +-
.../lib/apache/airavata/api/Airavata-remote | 105 +
.../lib/apache/airavata/api/Airavata.py | 18108 ++--
.../resources/lib/apache/airavata/api/ttypes.py | 1 +
.../model/appcatalog/computeresource/ttypes.py | 5 +
.../lib/apache/airavata/model/job/ttypes.py | 22 +-
.../lib/apache/airavata/model/process/ttypes.py | 42 +-
.../lib/apache/airavata/model/task/ttypes.py | 46 +-
.../computeresource/LOCALSubmission.java | 53 +-
.../appcatalog/computeresource/MonitorMode.java | 5 +-
.../credentialsummary/CredentialSummary.java | 826 +
.../UserComputeResourcePreference.java | 1381 +
.../UserResourceProfile.java | 1169 +
.../UserStoragePreference.java | 728 +
.../org/apache/airavata/model/job/JobModel.java | 103 +-
.../airavata/model/process/ProcessModel.java | 179 +-
.../apache/airavata/model/task/TaskModel.java | 178 +-
dev-tools/ansible/README.md | 15 +-
dev-tools/ansible/group_vars/all.yml | 5 +-
dev-tools/ansible/hosts | 2 +-
.../templates/airavata-server.properties.j2 | 13 +
.../templates/airavata-server.properties.j2 | 13 +
.../roles/pga/templates/pga_config.php.j2 | 2 +-
.../templates/airavata-server.properties.j2 | 13 +
modules/cloud/ansible-playbooks/README.md | 61 +-
.../roles/openstack/tasks/main.yml | 28 +
.../roles/openstack/vars/main.yml | 10 +
.../openstack/vars/openstack-credentials.yml | 10 +
modules/cloud/ansible-playbooks/site.yml | 5 +
modules/commons/pom.xml | 22 +-
.../airavata/common/utils/BuildConstant.java | 26 +
.../airavata/common/logging/Exception.java | 66 +
.../airavata/common/logging/LogEntry.java | 132 +
.../airavata/common/logging/MDCConstants.java | 30 +
.../apache/airavata/common/logging/MDCUtil.java | 32 +
.../airavata/common/logging/ServerId.java | 68 +
.../common/logging/kafka/KafkaAppender.java | 115 +
.../airavata/common/utils/AiravataUtils.java | 2 +-
.../airavata/common/utils/AiravataZKUtils.java | 13 +-
.../common/utils/ApplicationSettings.java | 34 +-
.../airavata/common/utils/AwsMetadata.java | 137 +
.../airavata/common/utils/ServerSettings.java | 34 +
.../apache/airavata/common/utils/XMLUtil.java | 586 -
.../airavata/common/utils/XmlFormatter.java | 82 -
.../airavata/common/utils/XMLUtilTest.java | 56 -
.../src/main/resources/FORK_Groovy.template | 13 +
.../server/src/main/resources/ForkTemplate.xslt | 24 -
.../server/src/main/resources/LSFTemplate.xslt | 92 -
.../src/main/resources/LSF_Groovy.template | 27 +
.../server/src/main/resources/PBSTemplate.xslt | 107 -
.../src/main/resources/PBS_Groovy.template | 31 +
.../src/main/resources/SLURMTemplate.xslt | 102 -
.../src/main/resources/SLURM_Groovy.template | 28 +
.../server/src/main/resources/UGETemplate.xslt | 78 -
.../src/main/resources/UGE_Groovy.template | 28 +
.../main/resources/airavata-server.properties | 27 +-
.../server/src/main/resources/gfac-config.yaml | 4 +-
.../server/src/main/resources/logback.xml | 31 +
.../credential/store/credential/Credential.java | 13 +
.../server/CredentialStoreServerHandler.java | 96 +-
.../store/store/impl/db/CredentialsDAO.java | 15 +-
.../store/store/impl/db/CredentialsDAOTest.java | 3 +-
.../store/store/impl/db/SSHCredentialTest.java | 2 +-
.../store/cpi/CredentialStoreService.java | 4086 +-
.../store/datamodel/CertificateCredential.java | 27 +-
.../store/datamodel/CommunityUser.java | 26 +-
.../store/datamodel/PasswordCredential.java | 27 +-
.../store/datamodel/SSHCredential.java | 142 +-
.../store/datamodel/SSHCredentialSummary.java | 914 +
.../exception/CredentialStoreException.java | 2 +-
modules/distribution/pom.xml | 75 +-
.../src/main/assembly/bin-assembly.xml | 24 +-
.../gfac/bes/provider/impl/BESProvider.java | 5 +-
.../impl/JSDLGeneratorTestWithMyProxyAuth.java | 2 +-
modules/gfac/gfac-core/pom.xml | 33 +-
.../apache/airavata/gfac/core/GFacUtils.java | 209 +-
.../apache/airavata/gfac/core/GroovyMap.java | 99 +
.../airavata/gfac/core/JobDescriptor.java | 489 -
.../org/apache/airavata/gfac/core/Script.java | 79 +
.../gfac/core/cluster/OutputParser.java | 8 -
.../gfac/core/context/ProcessContext.java | 20 +-
.../airavata/gfac/core/context/TaskContext.java | 2 +-
.../gfac/core/x2012/x12/AfterAnyList.java | 185 -
.../gfac/core/x2012/x12/AfterOKList.java | 185 -
.../gfac/core/x2012/x12/ExportProperties.java | 202 -
.../airavata/gfac/core/x2012/x12/InputList.java | 185 -
.../core/x2012/x12/JobDescriptorDocument.java | 131 -
.../gfac/core/x2012/x12/ModuleLoadCommands.java | 185 -
.../airavata/gfac/core/x2012/x12/PbsParams.java | 1500 -
.../gfac/core/x2012/x12/PostJobCommands.java | 185 -
.../gfac/core/x2012/x12/PreJobCommands.java | 185 -
.../core/x2012/x12/impl/AfterAnyListImpl.java | 254 -
.../core/x2012/x12/impl/AfterOKListImpl.java | 254 -
.../x2012/x12/impl/ExportPropertiesImpl.java | 252 -
.../gfac/core/x2012/x12/impl/InputListImpl.java | 254 -
.../x12/impl/JobDescriptorDocumentImpl.java | 96 -
.../x2012/x12/impl/ModuleLoadCommandsImpl.java | 254 -
.../gfac/core/x2012/x12/impl/PbsParamsImpl.java | 4381 -
.../x2012/x12/impl/PostJobCommandsImpl.java | 254 -
.../core/x2012/x12/impl/PreJobCommandsImpl.java | 254 -
.../src/main/resources/PBSJobDescriptor.xsd | 116 -
.../src/main/resources/gsissh-schemas.xsdconfig | 14 -
.../airavata/gfac/core/GFacUtilsTest.java | 13 +
.../org/apache/airavata/gfac/impl/Factory.java | 11 +-
.../airavata/gfac/impl/GFacEngineImpl.java | 16 +-
.../apache/airavata/gfac/impl/HTTPUtils.java | 7 +
.../airavata/gfac/impl/LocalRemoteCluster.java | 42 +-
.../gfac/impl/job/ForkOutputParser.java | 5 -
.../airavata/gfac/impl/job/LSFOutputParser.java | 10 +-
.../airavata/gfac/impl/job/PBSOutputParser.java | 90 +-
.../gfac/impl/job/SlurmOutputParser.java | 61 +-
.../airavata/gfac/impl/job/UGEOutputParser.java | 89 +-
.../gfac/impl/task/BESJobSubmissionTask.java | 5 +-
.../impl/task/DefaultJobSubmissionTask.java | 8 +-
.../gfac/impl/task/ForkJobSubmissionTask.java | 6 +-
.../gfac/impl/task/LocalJobSubmissionTask.java | 150 +-
.../airavata/gfac/impl/task/TaskUtils.java | 2 +-
.../gfac/monitor/email/EmailBasedMonitor.java | 6 +-
.../services/impl/BigRed2TestWithSSHAuth.java | 2 +-
.../impl/GSISSHProviderTestWithMyProxyAuth.java | 2 +-
.../impl/SSHProviderTestWithSSHAuth.java | 2 +-
.../airavata/gfac/server/GfacServerHandler.java | 41 +-
modules/group-manager/pom.xml | 58 -
.../internal/dao/PermissionEntryDAO.java | 286 -
.../dao/hib3/Hib3PermissionEntryDAO.java | 1123 -
.../grouper/permissions/PermissionFinder.java | 770 -
.../airavata/grouper/AiravataGrouperUtil.java | 35 -
.../airavata/grouper/GroupManagerCPI.java | 35 -
.../airavata/grouper/GroupManagerException.java | 36 -
.../airavata/grouper/GroupManagerFactory.java | 42 -
.../airavata/grouper/GroupManagerImpl.java | 118 -
.../apache/airavata/grouper/SubjectType.java | 14 -
.../apache/airavata/grouper/group/Group.java | 108 -
.../airavata/grouper/group/GroupMembership.java | 88 -
.../grouper/group/GroupMembershipType.java | 12 -
.../airavata/grouper/group/GroupService.java | 39 -
.../grouper/group/GroupServiceImpl.java | 412 -
.../grouper/permission/PermissionAction.java | 15 -
.../permission/PermissionServiceImpl.java | 86 -
.../airavata/grouper/resource/Resource.java | 145 -
.../resource/ResourceNotFoundException.java | 35 -
.../grouper/resource/ResourceServiceImpl.java | 343 -
.../airavata/grouper/resource/ResourceType.java | 51 -
.../airavata/grouper/role/RoleServiceImpl.java | 112 -
.../Owasp.CsrfGuard.overlay.properties | 78 -
.../main/resources/Owasp.CsrfGuard.properties | 403 -
.../group-manager/src/main/resources/README.txt | 21 -
.../src/main/resources/ehcache.xml | 1517 -
.../resources/grouper-loader.base.properties | 351 -
.../main/resources/grouper-loader.properties | 25 -
.../src/main/resources/grouper.base.properties | 1017 -
.../resources/grouper.client.base.properties | 552 -
.../main/resources/grouper.client.properties | 552 -
.../resources/grouper.hibernate.base.properties | 120 -
.../main/resources/grouper.hibernate.properties | 10 -
.../src/main/resources/grouper.properties | 1017 -
.../src/main/resources/grouperUtf8.txt | 1 -
.../src/main/resources/log4j.properties | 154 -
.../src/main/resources/morphString.properties | 21 -
.../src/main/resources/server.properties | 26 -
.../src/main/resources/sources.xml | 875 -
.../src/main/resources/spy.properties | 54 -
.../group-manager/src/main/resources/sqltool.rc | 6 -
.../src/main/resources/subject.base.properties | 46 -
.../src/main/resources/subject.properties | 27 -
.../cpi/impl/SimpleOrchestratorImpl.java | 12 +-
.../server/OrchestratorServerHandler.java | 33 +-
modules/registry/registry-core/pom.xml | 27 -
.../core/app/catalog/impl/AppCatalogImpl.java | 5 +
.../app/catalog/impl/ComputeResourceImpl.java | 1 +
.../catalog/impl/UsrResourceProfileImpl.java | 369 +
.../core/app/catalog/model/LocalSubmission.java | 11 +
.../model/UserComputeResourcePreference.java | 185 +
.../model/UserComputeResourcePreferencePK.java | 63 +
.../app/catalog/model/UserResourceProfile.java | 114 +
.../catalog/model/UserStoragePreference.java | 105 +
.../catalog/model/UserStoragePreferencePK.java | 64 +
.../resources/AppCatAbstractResource.java | 26 +-
.../ComputeHostPreferenceResource.java | 4 +
.../resources/LocalSubmissionResource.java | 12 +-
.../UserComputeHostPreferenceResource.java | 457 +
.../resources/UserResourceProfileResource.java | 454 +
.../UserStoragePreferenceResource.java | 378 +
.../app/catalog/util/AppCatalogJPAUtils.java | 73 +
.../catalog/util/AppCatalogResourceType.java | 3 +
.../util/AppCatalogThriftConversion.java | 64 +
.../experiment/catalog/ExpCatResourceUtils.java | 12 +-
.../catalog/impl/ExperimentRegistry.java | 8 +
.../src/main/resources/META-INF/persistence.xml | 3 +
.../src/main/resources/appcatalog-derby.sql | 1 +
.../src/main/resources/appcatalog-mysql.sql | 1 +
.../src/main/resources/credstore-mysql.sql | 10 +-
.../app/catalog/UserResourceProfileTest.java | 129 +
.../airavata/registry/cpi/AppCatalog.java | 7 +
.../registry/cpi/UsrResourceProfile.java | 90 +
.../service/handler/RegistryServerHandler.java | 568 +-
.../airavata/registry/api/RegistryService.java | 21916 ++++-
.../api/exception/RegistryServiceException.java | 2 +-
.../org/apache/airavata/server/ServerMain.java | 39 +-
modules/sharing-registry/README.md | 3 +
modules/sharing-registry/pom.xml | 31 +
.../sharing-data-migrator/pom.xml | 22 +
.../migrator/airavata/AiravataDataMigrator.java | 162 +
.../migrator/airavata/ConnectionFactory.java | 59 +
.../sharing-registry-distribution/pom.xml | 91 +
.../src/main/assembly/bin-assembly.xml | 70 +
.../src/main/resources/bin/setenv.sh | 61 +
.../src/main/resources/bin/sharing-registry.sh | 90 +
.../sharing-registry-server/pom.xml | 112 +
.../registry/db/entities/DomainEntity.java | 113 +
.../registry/db/entities/EntityEntity.java | 202 +
.../sharing/registry/db/entities/EntityPK.java | 76 +
.../registry/db/entities/EntityTypeEntity.java | 126 +
.../registry/db/entities/EntityTypePK.java | 76 +
.../db/entities/GroupMembershipEntity.java | 123 +
.../registry/db/entities/GroupMembershipPK.java | 88 +
.../db/entities/PermissionTypeEntity.java | 126 +
.../registry/db/entities/PermissionTypePK.java | 76 +
.../registry/db/entities/SharingEntity.java | 147 +
.../sharing/registry/db/entities/SharingPK.java | 114 +
.../registry/db/entities/UserEntity.java | 158 +
.../registry/db/entities/UserGroupEntity.java | 162 +
.../registry/db/entities/UserGroupPK.java | 75 +
.../sharing/registry/db/entities/UserPK.java | 75 +
.../db/repositories/AbstractRepository.java | 138 +
.../db/repositories/DomainRepository.java | 34 +
.../db/repositories/EntityRepository.java | 149 +
.../db/repositories/EntityTypeRepository.java | 35 +
.../repositories/GroupMembershipRepository.java | 79 +
.../repositories/PermissionTypeRepository.java | 53 +
.../db/repositories/SharingRepository.java | 81 +
.../db/repositories/UserGroupRepository.java | 54 +
.../db/repositories/UserRepository.java | 54 +
.../sharing/registry/db/utils/Committer.java | 27 +
.../registry/db/utils/ConnectionPool.java | 382 +
.../sharing/registry/db/utils/DBConstants.java | 106 +
.../registry/db/utils/DatabaseCreator.java | 353 +
.../sharing/registry/db/utils/JPAUtils.java | 230 +
.../sharing/registry/db/utils/JdbcStorage.java | 175 +
.../db/utils/ObjectMapperSingleton.java | 39 +
.../sharing/registry/server/ServerMain.java | 102 +
.../registry/server/SharingRegistryServer.java | 147 +
.../server/SharingRegistryServerHandler.java | 947 +
.../src/main/resources/META-INF/persistence.xml | 15 +
.../main/resources/sharing-registry-derby.sql | 138 +
.../main/resources/sharing-registry-mysql.sql | 138 +
.../SharingRegistryServerHandlerTest.java | 278 +
.../registry/SharingRegistryServiceTest.java | 314 +
.../sharing/registry/util/Initialize.java | 298 +
.../sharing-registry-stubs/pom.xml | 56 +
.../sharing/registry/models/Domain.java | 811 +
.../sharing/registry/models/Entity.java | 1583 +
.../registry/models/EntitySearchField.java | 70 +
.../sharing/registry/models/EntityType.java | 931 +
.../registry/models/GroupCardinality.java | 50 +
.../sharing/registry/models/GroupChildType.java | 49 +
.../registry/models/GroupMembership.java | 941 +
.../sharing/registry/models/GroupType.java | 49 +
.../sharing/registry/models/PermissionType.java | 931 +
.../registry/models/SearchCondition.java | 63 +
.../sharing/registry/models/SearchCriteria.java | 649 +
.../sharing/registry/models/Sharing.java | 1155 +
.../models/SharingRegistryException.java | 395 +
.../sharing/registry/models/SharingType.java | 52 +
.../airavata/sharing/registry/models/User.java | 1266 +
.../sharing/registry/models/UserGroup.java | 1287 +
.../models/sharing_modelsConstants.java | 41 +
.../service/cpi/SharingRegistryService.java | 49079 +++++++++++
.../sharing-service-docs/api-docs/index.html | 79 +
.../api-docs/sharing_cpi.html | 297 +
.../api-docs/sharing_models.html | 283 +
.../sharing-service-docs/api-docs/style.css | 184 +
.../sharing-service-docs/index.html | 362 +
.../sharing-service-docs/sharing_overview.png | Bin 0 -> 49205 bytes
.../sharing-service-docs/style.css | 184 +
.../thrift_models/sharing_cpi.thrift | 211 +
.../thrift_models/sharing_models.thrift | 274 +
.../thrift_models/thrift-gen.sh | 22 +
.../utils/PropertyReader.java | 3 +-
.../core/AbstractThriftDeserializer.java | 3 +-
.../engine/interpretor/WorkflowInterpreter.java | 1 -
.../invoker/MsgBoxWsaResponsesCorrelator.java | 2 +-
.../component/ws/WSComponentApplication.java | 1 -
.../workflow/model/gpel/script/BPELScript.java | 2 -
.../model/gpel/script/WorkflowWSDL.java | 1 -
.../airavata/workflow/model/wf/Workflow.java | 1 -
modules/workflow/workflow-core/pom.xml | 11 +-
.../apache/airavata/workflow/core/XMLUtil.java | 587 +
.../airavata/workflow/core/XmlFormatter.java | 82 +
.../airavata/workflow/core/XMLUtilTest.java | 56 +
pom.xml | 69 +-
.../airavata-apis/airavata_api.thrift | 426 +-
.../component-cpis/credential-store-cpi.thrift | 9 +-
.../credential_store_data_models.thrift | 62 -
.../component-cpis/registry-api.thrift | 316 +
.../credential_store_data_models.thrift | 72 +
.../compute_resource_model.thrift | 5 +-
.../credential_summary.thrift | 32 +
.../resource_catalog_models.thrift | 4 +-
.../user_resource_profile_model.thrift | 105 +
.../generate-thrift-stubs.sh | 24 +
324 files changed, 209896 insertions(+), 79409 deletions(-)
----------------------------------------------------------------------
[6/6] airavata git commit: Start aurora cloud monitoring service
Posted by sh...@apache.org.
Start aurora cloud monitoring service
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/6d51651c
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/6d51651c
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/6d51651c
Branch: refs/heads/develop
Commit: 6d51651ca45920277ab327e676a080103d3fd9fd
Parents: 58db743
Author: Shameera Rathnayaka <sh...@gmail.com>
Authored: Mon Oct 31 20:10:15 2016 -0400
Committer: Shameera Rathnayaka <sh...@gmail.com>
Committed: Mon Oct 31 20:10:15 2016 -0400
----------------------------------------------------------------------
.../org/apache/airavata/gfac/impl/Factory.java | 37 +++++++++++++++-----
.../airavata/gfac/impl/GFacEngineImpl.java | 2 ++
.../cpi/impl/SimpleOrchestratorImpl.java | 4 ++-
3 files changed, 33 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/6d51651c/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java
index a808ad3..8dd8699 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java
@@ -62,6 +62,7 @@ import org.apache.airavata.gfac.impl.job.UGEJobConfiguration;
import org.apache.airavata.gfac.impl.task.ArchiveTask;
import org.apache.airavata.gfac.impl.watcher.CancelRequestWatcherImpl;
import org.apache.airavata.gfac.impl.watcher.RedeliveryRequestWatcherImpl;
+import org.apache.airavata.gfac.monitor.cloud.AuroraJobMonitor;
import org.apache.airavata.gfac.monitor.email.EmailBasedMonitor;
import org.apache.airavata.messaging.core.MessageHandler;
import org.apache.airavata.messaging.core.MessagingFactory;
@@ -184,6 +185,9 @@ public abstract class Factory {
}
public static JobManagerConfiguration getJobManagerConfiguration(ResourceJobManager resourceJobManager) throws GFacException {
+ if(resourceJobManager == null)
+ return null;
+
ResourceConfig resourceConfig = Factory.getResourceConfig(resourceJobManager.getResourceJobManagerType());
OutputParser outputParser;
try {
@@ -208,9 +212,9 @@ public abstract class Factory {
case UGE:
return new UGEJobConfiguration("UGE_Groovy.template", ".pbs", resourceJobManager.getJobManagerBinPath(),
resourceJobManager.getJobManagerCommands(), outputParser);
- case FORK:
- return new ForkJobConfiguration("FORK_Groovy.template", ".sh", resourceJobManager.getJobManagerBinPath(),
- resourceJobManager.getJobManagerCommands(), outputParser);
+ case FORK:
+ return new ForkJobConfiguration("FORK_Groovy.template", ".sh", resourceJobManager.getJobManagerBinPath(),
+ resourceJobManager.getJobManagerCommands(), outputParser);
default:
return null;
}
@@ -244,17 +248,23 @@ public abstract class Factory {
jobSubmissionProtocol == JobSubmissionProtocol.LOCAL_FORK) {
remoteCluster = new LocalRemoteCluster(processContext.getServerInfo(), jobManagerConfiguration, null);
} else if (jobSubmissionProtocol == JobSubmissionProtocol.SSH ||
- jobSubmissionProtocol == JobSubmissionProtocol.SSH_FORK) {
+ jobSubmissionProtocol == JobSubmissionProtocol.SSH_FORK
+ || jobSubmissionProtocol == JobSubmissionProtocol.CLOUD) {
+
remoteCluster = new HPCRemoteCluster(processContext.getServerInfo(), jobManagerConfiguration,
processContext.getSshKeyAuthentication());
- }
+ }else {
+ throw new GFacException("No remote cluster implementation map to job submission protocol "
+ + jobSubmissionProtocol.name());
+ }
remoteClusterMap.put(key, remoteCluster);
}else {
AuthenticationInfo authentication = remoteCluster.getAuthentication();
if (authentication instanceof SSHKeyAuthentication){
SSHKeyAuthentication sshKeyAuthentication = (SSHKeyAuthentication)authentication;
if (!sshKeyAuthentication.getUserName().equals(getLoginUserName(processContext))){
- JobManagerConfiguration jobManagerConfiguration = getJobManagerConfiguration(processContext.getResourceJobManager());
+ JobManagerConfiguration jobManagerConfiguration =
+ getJobManagerConfiguration(processContext.getResourceJobManager());
if (jobSubmissionProtocol == JobSubmissionProtocol.SSH ||
jobSubmissionProtocol == JobSubmissionProtocol.SSH_FORK) {
remoteCluster = new HPCRemoteCluster(processContext.getServerInfo(), jobManagerConfiguration,
@@ -281,7 +291,10 @@ public abstract class Factory {
} else if (dataMovementProtocol == DataMovementProtocol.SCP) {
remoteCluster = new HPCRemoteCluster(processContext.getServerInfo(), jobManagerConfiguration,
processContext.getSshKeyAuthentication());
- }
+ }else {
+ throw new GFacException("No remote cluster implementation map to job data movement protocol "
+ + dataMovementProtocol.name());
+ }
remoteClusterMap.put(key, remoteCluster);
}else {
@@ -289,7 +302,8 @@ public abstract class Factory {
if (authentication instanceof SSHKeyAuthentication){
SSHKeyAuthentication sshKeyAuthentication = (SSHKeyAuthentication)authentication;
if (!sshKeyAuthentication.getUserName().equals(getLoginUserName(processContext))){
- JobManagerConfiguration jobManagerConfiguration = getJobManagerConfiguration(processContext.getResourceJobManager());
+ JobManagerConfiguration jobManagerConfiguration =
+ getJobManagerConfiguration(processContext.getResourceJobManager());
dataMovementProtocol = processContext.getDataMovementProtocol();
if (dataMovementProtocol == DataMovementProtocol.SCP) {
remoteCluster = new HPCRemoteCluster(processContext.getServerInfo(), jobManagerConfiguration,
@@ -450,8 +464,13 @@ public abstract class Factory {
case JOB_EMAIL_NOTIFICATION_MONITOR:
EmailBasedMonitor emailBasedMonitor = new EmailBasedMonitor(Factory.getResourceConfig());
jobMonitorServices.put(MonitorMode.JOB_EMAIL_NOTIFICATION_MONITOR, emailBasedMonitor);
- jobMonitor = ((JobMonitor) emailBasedMonitor);
+ jobMonitor = emailBasedMonitor;
new Thread(emailBasedMonitor).start();
+ break;
+ case CLOUD_JOB_MONITOR:
+ AuroraJobMonitor auroraJobMonitor = AuroraJobMonitor.getInstance();
+ new Thread(auroraJobMonitor).start();
+ jobMonitorServices.put(MonitorMode.CLOUD_JOB_MONITOR, auroraJobMonitor);
}
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6d51651c/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/GFacEngineImpl.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/GFacEngineImpl.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/GFacEngineImpl.java
index 6f8e3ad..212b246 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/GFacEngineImpl.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/GFacEngineImpl.java
@@ -929,6 +929,8 @@ public class GFacEngineImpl implements GFacEngine {
(jsInterface.getJobSubmissionInterfaceId());
processCtx.setMonitorMode(sshJobSubmission.getMonitorMode()); // fixme - Move this to populate process
resourceJobManager = sshJobSubmission.getResourceJobManager();
+ } else if (jsInterface.getJobSubmissionProtocol() == JobSubmissionProtocol.CLOUD) {
+ return null;
} else {
throw new GFacException("Unsupported JobSubmissionProtocol - " + jsInterface.getJobSubmissionProtocol()
.name());
http://git-wip-us.apache.org/repos/asf/airavata/blob/6d51651c/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
index b97e79a..66bf5ca 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
@@ -291,7 +291,9 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{
// TODO - breakdown unicore all in one task to multiple tasks, then we don't need to handle UNICORE here.
taskIdList.addAll(createAndSaveSubmissionTasks(gatewayId, preferredJobSubmissionInterface, processModel, userGivenWallTime));
} else {
- taskIdList.addAll(createAndSaveEnvSetupTask(gatewayId, processModel, experimentCatalog));
+ if(resourcePreference.getPreferredJobSubmissionProtocol() != JobSubmissionProtocol.CLOUD){
+ taskIdList.addAll(createAndSaveEnvSetupTask(gatewayId, processModel, experimentCatalog));
+ }
taskIdList.addAll(createAndSaveInputDataStagingTasks(processModel, gatewayId));
if (autoSchedule) {
List<BatchQueue> definedBatchQueues = computeResource.getBatchQueues();
[5/6] airavata git commit: Merge pull request 69
Posted by sh...@apache.org.
Merge pull request 69
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/58db7434
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/58db7434
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/58db7434
Branch: refs/heads/develop
Commit: 58db74346b1dd9d3e921b8e8ce98ef4c067854d6
Parents: 21b1923 b7bebec
Author: Shameera Rathnayaka <sh...@gmail.com>
Authored: Mon Oct 31 19:40:25 2016 -0400
Committer: Shameera Rathnayaka <sh...@gmail.com>
Committed: Mon Oct 31 19:40:25 2016 -0400
----------------------------------------------------------------------
.../cloud/aurora/sample/AuroraClientSample.java | 56 +++++++++++++++++---
1 file changed, 50 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
[3/6] airavata git commit: Adding Autodock test job submission
Posted by sh...@apache.org.
Adding Autodock test job submission
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/b7bebec4
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/b7bebec4
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/b7bebec4
Branch: refs/heads/develop
Commit: b7bebec439cbdd1e63f3b0437a4f2b9ee3995c44
Parents: b66fa88
Author: Gourav Shenoy <sh...@gmail.com>
Authored: Mon Oct 31 16:32:34 2016 -0400
Committer: Gourav Shenoy <sh...@gmail.com>
Committed: Mon Oct 31 16:32:34 2016 -0400
----------------------------------------------------------------------
.../cloud/aurora/sample/AuroraClientSample.java | 56 +++++++++++++++++---
1 file changed, 50 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/b7bebec4/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/sample/AuroraClientSample.java
----------------------------------------------------------------------
diff --git a/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/sample/AuroraClientSample.java b/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/sample/AuroraClientSample.java
index 44d0ecf..3f29969 100644
--- a/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/sample/AuroraClientSample.java
+++ b/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/sample/AuroraClientSample.java
@@ -21,8 +21,10 @@ package org.apache.airavata.cloud.aurora.sample;
import java.text.MessageFormat;
import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.Properties;
import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
import org.apache.airavata.cloud.aurora.client.AuroraSchedulerClientFactory;
import org.apache.airavata.cloud.aurora.client.AuroraThriftClient;
@@ -129,6 +131,43 @@ public class AuroraClientSample {
System.out.println(response);
}
+ public static void createAutoDockJob() throws Exception {
+ JobKeyBean jobKey = new JobKeyBean("devel", "centos", "test_autodock");
+ IdentityBean owner = new IdentityBean("centos");
+
+ String working_dir = "/home/centos/efs-mount-point/job_" + ThreadLocalRandom.current().nextInt(1, 101) + "/";
+ String autodock_path = "/home/centos/efs-mount-point/autodock-vina";
+ ProcessBean proc1 = new ProcessBean("process_1", "mkdir " + working_dir, false);
+ ProcessBean proc2 = new ProcessBean("process_2", "cp " + autodock_path + "/vina_screenM.sh " + working_dir, false);
+ ProcessBean proc3 = new ProcessBean("process_3", "cp " + autodock_path + "/ligand* " + working_dir, false);
+ ProcessBean proc4 = new ProcessBean("process_4", "cd " + working_dir + " && sh vina_screenM.sh", false);
+
+ Set<ProcessBean> processes = new LinkedHashSet<>();
+ processes.add(proc1);
+ processes.add(proc2);
+ processes.add(proc3);
+ processes.add(proc4);
+
+ ResourceBean resources = new ResourceBean(1.5, 125, 512);
+
+ TaskConfigBean taskConfig = new TaskConfigBean("test_autodock", processes, resources);
+ JobConfigBean jobConfig = new JobConfigBean(jobKey, owner, taskConfig, "example");
+
+ String executorConfigJson = AuroraThriftClientUtil.getExecutorConfigJson(jobConfig);
+ System.out.println(executorConfigJson);
+
+ AuroraThriftClient client = AuroraThriftClient.getAuroraThriftClient(Constants.AURORA_SCHEDULER_PROP_FILE);
+ ResponseBean response = client.createJob(jobConfig);
+ System.out.println(response);
+ }
+
+ public static void killTasks(String jobName) throws Exception {
+ JobKeyBean jobKey = new JobKeyBean("devel", "centos", jobName);
+ AuroraThriftClient client = AuroraThriftClient.getAuroraThriftClient(Constants.AURORA_SCHEDULER_PROP_FILE);
+ ResponseBean response = client.killTasks(jobKey, new HashSet<>());
+ System.out.println(response);
+ }
+
/**
* The main method.
*
@@ -141,14 +180,19 @@ public class AuroraClientSample {
String auroraPort = properties.getProperty(Constants.AURORA_SCHEDULER_PORT);
auroraSchedulerClient = AuroraSchedulerClientFactory.createReadOnlySchedulerClient(MessageFormat.format(Constants.AURORA_SCHEDULER_CONNECTION_URL, auroraHost, auroraPort));
- // get jobs summary
-// AuroraClientSample.getJobSummary(auroraSchedulerClient);
-
// create sample job
// AuroraClientSample.createJob();
- AuroraThriftClient client = AuroraThriftClient.getAuroraThriftClient(Constants.AURORA_SCHEDULER_PROP_FILE);
- ResponseBean response = client.getPendingReasonForJob(new JobKeyBean("devel", "centos", "hello_pending"));
- System.out.println(response);
+ AuroraClientSample.createAutoDockJob();
+
+ // kill pending job
+// AuroraClientSample.killTasks("test_autodock");
+
+ // get jobs summary
+ AuroraClientSample.getJobSummary(auroraSchedulerClient);
+
+// AuroraThriftClient client = AuroraThriftClient.getAuroraThriftClient(Constants.AURORA_SCHEDULER_PROP_FILE);
+// ResponseBean response = client.getPendingReasonForJob(new JobKeyBean("devel", "centos", "hello_pending"));
+// System.out.println(response);
} catch (Exception ex) {
ex.printStackTrace();
}
[4/6] airavata git commit: Added job submission task and aurora
montoring services
Posted by sh...@apache.org.
Added job submission task and aurora montoring services
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/21b1923d
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/21b1923d
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/21b1923d
Branch: refs/heads/develop
Commit: 21b1923da2184d6df71a855caf9aea80ce33b300
Parents: 325d163
Author: Shameera Rathnayaka <sh...@gmail.com>
Authored: Mon Oct 31 19:36:17 2016 -0400
Committer: Shameera Rathnayaka <sh...@gmail.com>
Committed: Mon Oct 31 19:36:17 2016 -0400
----------------------------------------------------------------------
modules/cloud/aurora-client/pom.xml | 8 +
.../cloud/aurora/client/AuroraThriftClient.java | 2 +-
.../main/resources/aurora-scheduler.properties | 2 +-
modules/cloud/cloud-provisioning/pom.xml | 7 +
.../apache/airavata/gfac/core/GFacUtils.java | 88 ++++++-
modules/gfac/gfac-impl/pom.xml | 5 +
.../apache/airavata/gfac/impl/AuroraUtils.java | 30 +++
.../gfac/impl/task/AuroraJobSubmission.java | 146 +++++++++++
.../gfac/monitor/cloud/AuroraJobMonitor.java | 247 +++++++++++++++++++
.../core/utils/OrchestratorUtils.java | 16 ++
.../cpi/impl/SimpleOrchestratorImpl.java | 5 +-
pom.xml | 2 +-
12 files changed, 543 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/21b1923d/modules/cloud/aurora-client/pom.xml
----------------------------------------------------------------------
diff --git a/modules/cloud/aurora-client/pom.xml b/modules/cloud/aurora-client/pom.xml
index 4cac7d9..8188c49 100644
--- a/modules/cloud/aurora-client/pom.xml
+++ b/modules/cloud/aurora-client/pom.xml
@@ -70,7 +70,15 @@
<directory>src/test/resources</directory>
</testResource>
</testResources>
+
<plugins>
+ <plugin>
+ <groupId>com.mycila</groupId>
+ <artifactId>license-maven-plugin</artifactId>
+ <configuration>
+ <header>../../../apache-license-header.txt</header>
+ </configuration>
+ </plugin>
</plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/airavata/blob/21b1923d/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/client/AuroraThriftClient.java
----------------------------------------------------------------------
diff --git a/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/client/AuroraThriftClient.java b/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/client/AuroraThriftClient.java
index 3fb2468..0e0b36e 100644
--- a/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/client/AuroraThriftClient.java
+++ b/modules/cloud/aurora-client/src/main/java/org/apache/airavata/cloud/aurora/client/AuroraThriftClient.java
@@ -193,7 +193,7 @@ public class AuroraThriftClient {
* @return the job details
* @throws Exception the exception
*/
- public ResponseBean getJobDetails(JobKeyBean jobKeyBean) throws Exception {
+ public JobDetailsResponseBean getJobDetails(JobKeyBean jobKeyBean) throws Exception {
JobDetailsResponseBean response = null;
try {
if(jobKeyBean != null) {
http://git-wip-us.apache.org/repos/asf/airavata/blob/21b1923d/modules/cloud/aurora-client/src/main/resources/aurora-scheduler.properties
----------------------------------------------------------------------
diff --git a/modules/cloud/aurora-client/src/main/resources/aurora-scheduler.properties b/modules/cloud/aurora-client/src/main/resources/aurora-scheduler.properties
index 6cb1fe9..0e1cc95 100644
--- a/modules/cloud/aurora-client/src/main/resources/aurora-scheduler.properties
+++ b/modules/cloud/aurora-client/src/main/resources/aurora-scheduler.properties
@@ -1,6 +1,6 @@
#Aurora scheduler properties
-aurora.scheduler.host=mesos-master-1
+aurora.scheduler.host=52.15.129.208
aurora.scheduler.port=8081
http://git-wip-us.apache.org/repos/asf/airavata/blob/21b1923d/modules/cloud/cloud-provisioning/pom.xml
----------------------------------------------------------------------
diff --git a/modules/cloud/cloud-provisioning/pom.xml b/modules/cloud/cloud-provisioning/pom.xml
index c08cfb9..7981a10 100644
--- a/modules/cloud/cloud-provisioning/pom.xml
+++ b/modules/cloud/cloud-provisioning/pom.xml
@@ -76,6 +76,13 @@
</testResource>
</testResources>
<plugins>
+ <plugin>
+ <groupId>com.mycila</groupId>
+ <artifactId>license-maven-plugin</artifactId>
+ <configuration>
+ <header>../../../apache-license-header.txt</header>
+ </configuration>
+ </plugin>
</plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/airavata/blob/21b1923d/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
index b69764e..66998c3 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
@@ -21,10 +21,14 @@ package org.apache.airavata.gfac.core;
import groovy.lang.Writable;
import groovy.text.GStringTemplateEngine;
-import groovy.text.SimpleTemplateEngine;
import groovy.text.TemplateEngine;
import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.*;
+import org.apache.airavata.common.utils.AiravataUtils;
+import org.apache.airavata.common.utils.AiravataZKUtils;
+import org.apache.airavata.common.utils.ApplicationSettings;
+import org.apache.airavata.common.utils.DBUtil;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.common.utils.ZkConstants;
import org.apache.airavata.credential.store.store.CredentialReader;
import org.apache.airavata.credential.store.store.impl.CredentialReaderImpl;
import org.apache.airavata.gfac.core.context.ProcessContext;
@@ -32,23 +36,54 @@ import org.apache.airavata.gfac.core.context.TaskContext;
import org.apache.airavata.messaging.core.MessageContext;
import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription;
import org.apache.airavata.model.appcatalog.appdeployment.CommandObject;
-import org.apache.airavata.model.appcatalog.computeresource.*;
+import org.apache.airavata.model.appcatalog.computeresource.CloudJobSubmission;
+import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
+import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionInterface;
+import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
+import org.apache.airavata.model.appcatalog.computeresource.LOCALSubmission;
+import org.apache.airavata.model.appcatalog.computeresource.MonitorMode;
+import org.apache.airavata.model.appcatalog.computeresource.ResourceJobManager;
+import org.apache.airavata.model.appcatalog.computeresource.ResourceJobManagerType;
+import org.apache.airavata.model.appcatalog.computeresource.SSHJobSubmission;
+import org.apache.airavata.model.appcatalog.computeresource.UnicoreJobSubmission;
import org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference;
import org.apache.airavata.model.application.io.DataType;
import org.apache.airavata.model.application.io.InputDataObjectType;
import org.apache.airavata.model.application.io.OutputDataObjectType;
import org.apache.airavata.model.commons.ErrorModel;
-import org.apache.airavata.model.data.replica.*;
+import org.apache.airavata.model.data.replica.DataProductModel;
+import org.apache.airavata.model.data.replica.DataProductType;
+import org.apache.airavata.model.data.replica.DataReplicaLocationModel;
+import org.apache.airavata.model.data.replica.ReplicaLocationCategory;
+import org.apache.airavata.model.data.replica.ReplicaPersistentType;
import org.apache.airavata.model.experiment.ExperimentModel;
import org.apache.airavata.model.job.JobModel;
-import org.apache.airavata.model.messaging.event.*;
+import org.apache.airavata.model.messaging.event.JobIdentifier;
+import org.apache.airavata.model.messaging.event.JobStatusChangeEvent;
+import org.apache.airavata.model.messaging.event.MessageType;
+import org.apache.airavata.model.messaging.event.ProcessIdentifier;
+import org.apache.airavata.model.messaging.event.ProcessStatusChangeEvent;
+import org.apache.airavata.model.messaging.event.TaskIdentifier;
+import org.apache.airavata.model.messaging.event.TaskStatusChangeEvent;
import org.apache.airavata.model.parallelism.ApplicationParallelismType;
import org.apache.airavata.model.process.ProcessModel;
import org.apache.airavata.model.scheduling.ComputationalResourceSchedulingModel;
-import org.apache.airavata.model.status.*;
+import org.apache.airavata.model.status.JobStatus;
+import org.apache.airavata.model.status.ProcessState;
+import org.apache.airavata.model.status.ProcessStatus;
+import org.apache.airavata.model.status.TaskState;
+import org.apache.airavata.model.status.TaskStatus;
import org.apache.airavata.model.task.JobSubmissionTaskModel;
import org.apache.airavata.registry.core.experiment.catalog.impl.RegistryFactory;
-import org.apache.airavata.registry.cpi.*;
+import org.apache.airavata.registry.cpi.AppCatalog;
+import org.apache.airavata.registry.cpi.AppCatalogException;
+import org.apache.airavata.registry.cpi.CompositeIdentifier;
+import org.apache.airavata.registry.cpi.ExpCatChildDataType;
+import org.apache.airavata.registry.cpi.ExperimentCatalog;
+import org.apache.airavata.registry.cpi.ExperimentCatalogModelType;
+import org.apache.airavata.registry.cpi.GwyResourceProfile;
+import org.apache.airavata.registry.cpi.RegistryException;
+import org.apache.airavata.registry.cpi.ReplicaCatalog;
import org.apache.airavata.registry.cpi.utils.Constants;
import org.apache.commons.io.FileUtils;
import org.apache.curator.framework.CuratorFramework;
@@ -65,15 +100,35 @@ import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
-import javax.xml.xpath.*;
-import java.io.*;
+import javax.xml.xpath.XPath;
+import javax.xml.xpath.XPathConstants;
+import javax.xml.xpath.XPathExpression;
+import javax.xml.xpath.XPathExpressionException;
+import javax.xml.xpath.XPathFactory;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
import java.net.InetAddress;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.security.SecureRandom;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -374,6 +429,17 @@ public class GFacUtils {
}
}
+ public static CloudJobSubmission getCloudJobSubmission(String submissionId) throws RegistryException {
+ try {
+ AppCatalog appCatalog = RegistryFactory.getAppCatalog();
+ return appCatalog.getComputeResource().getCloudJobSubmission(submissionId);
+ } catch (Exception e) {
+ String errorMsg = "Error while retrieving SSH job submission with submission id : " + submissionId;
+ log.error(errorMsg, e);
+ throw new RegistryException(errorMsg, e);
+ }
+ }
+
/**
* To convert list to separated value
*
@@ -700,7 +766,7 @@ public class GFacUtils {
return null;
}
- private static int generateJobName() {
+ public static int generateJobName() {
Random random = new Random();
int i = random.nextInt(Integer.MAX_VALUE);
i = i + 99999999;
http://git-wip-us.apache.org/repos/asf/airavata/blob/21b1923d/modules/gfac/gfac-impl/pom.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/pom.xml b/modules/gfac/gfac-impl/pom.xml
index 2a0a949..60e3259 100644
--- a/modules/gfac/gfac-impl/pom.xml
+++ b/modules/gfac/gfac-impl/pom.xml
@@ -122,5 +122,10 @@
<artifactId>commons-httpclient</artifactId>
<version>3.1</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>aurora-client</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/airavata/blob/21b1923d/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/AuroraUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/AuroraUtils.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/AuroraUtils.java
new file mode 100644
index 0000000..4412694
--- /dev/null
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/AuroraUtils.java
@@ -0,0 +1,30 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.airavata.gfac.impl;
+
+public class AuroraUtils {
+
+ public static final String ENVIRONMENT = "devel";
+ public static final String ROLE = "centos";
+ public static final String AURORA_SCHEDULER_PROP_FILE = "aurora-scheduler.properties";
+ public static final String CLUSTER = "example";
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/21b1923d/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/AuroraJobSubmission.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/AuroraJobSubmission.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/AuroraJobSubmission.java
new file mode 100644
index 0000000..c2e12d5
--- /dev/null
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/AuroraJobSubmission.java
@@ -0,0 +1,146 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.airavata.gfac.impl.task;
+
+import org.apache.airavata.cloud.aurora.client.AuroraThriftClient;
+import org.apache.airavata.cloud.aurora.client.bean.IdentityBean;
+import org.apache.airavata.cloud.aurora.client.bean.JobConfigBean;
+import org.apache.airavata.cloud.aurora.client.bean.JobKeyBean;
+import org.apache.airavata.cloud.aurora.client.bean.ProcessBean;
+import org.apache.airavata.cloud.aurora.client.bean.ResourceBean;
+import org.apache.airavata.cloud.aurora.client.bean.ResponseBean;
+import org.apache.airavata.cloud.aurora.client.bean.TaskConfigBean;
+import org.apache.airavata.cloud.aurora.util.AuroraThriftClientUtil;
+import org.apache.airavata.cloud.aurora.util.Constants;
+import org.apache.airavata.common.utils.AiravataUtils;
+import org.apache.airavata.gfac.core.GFacException;
+import org.apache.airavata.gfac.core.GFacUtils;
+import org.apache.airavata.gfac.core.context.ProcessContext;
+import org.apache.airavata.gfac.core.context.TaskContext;
+import org.apache.airavata.gfac.core.task.JobSubmissionTask;
+import org.apache.airavata.gfac.core.task.TaskException;
+import org.apache.airavata.gfac.impl.AuroraUtils;
+import org.apache.airavata.model.commons.ErrorModel;
+import org.apache.airavata.model.job.JobModel;
+import org.apache.airavata.model.status.JobState;
+import org.apache.airavata.model.status.JobStatus;
+import org.apache.airavata.model.status.TaskState;
+import org.apache.airavata.model.status.TaskStatus;
+import org.apache.airavata.model.task.TaskTypes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+
+public class AuroraJobSubmission implements JobSubmissionTask{
+
+ private static final Logger log = LoggerFactory.getLogger(AuroraJobSubmission.class);
+
+ @Override
+ public JobStatus cancel(TaskContext taskcontext) throws TaskException {
+ return null;
+ }
+
+ @Override
+ public void init(Map<String, String> propertyMap) throws TaskException {
+
+ }
+
+ @Override
+ public TaskStatus execute(TaskContext taskContext) {
+ TaskStatus taskStatus = new TaskStatus(TaskState.COMPLETED); // set to completed.
+ ProcessContext processContext = taskContext.getParentProcessContext();
+ JobModel jobModel = processContext.getJobModel();
+ jobModel.setTaskId(taskContext.getTaskId());
+ String jobIdAndName = "A" + GFacUtils.generateJobName();
+ jobModel.setJobName(jobIdAndName);
+ JobStatus jobStatus = new JobStatus();
+ jobStatus.setJobState(JobState.SUBMITTED);
+
+ try {
+ JobKeyBean jobKey = new JobKeyBean(AuroraUtils.ENVIRONMENT, AuroraUtils.ROLE, jobIdAndName);
+ IdentityBean owner = new IdentityBean(AuroraUtils.ROLE);
+ // only autodoc vina
+ String workingDir = taskContext.getWorkingDir();
+ ProcessBean proc1 = new ProcessBean("process_1", "mkdir -p " + workingDir, false);
+ ProcessBean proc2 = new ProcessBean("process_1", "cp -rf /home/centos/efs-mount-point/autodock-vina/* " + workingDir , false);
+ ProcessBean proc3 = new ProcessBean("process_2", "cd " + workingDir + " && ./vina_screenA.sh", false);
+ Set<ProcessBean> processes = new LinkedHashSet<>();
+ processes.add(proc1);
+ processes.add(proc2);
+ processes.add(proc3);
+
+ ResourceBean resources = new ResourceBean(1.5, 512, 512);
+
+ TaskConfigBean taskConfig = new TaskConfigBean("Airavata-Aurora-" + jobIdAndName, processes, resources);
+ JobConfigBean jobConfig = new JobConfigBean(jobKey, owner, taskConfig, AuroraUtils.CLUSTER);
+
+ String executorConfigJson = AuroraThriftClientUtil.getExecutorConfigJson(jobConfig);
+ log.info("Executor Config for Job {} , {}", jobIdAndName, executorConfigJson);
+
+ AuroraThriftClient client = AuroraThriftClient.getAuroraThriftClient(Constants.AURORA_SCHEDULER_PROP_FILE);
+ ResponseBean response = client.createJob(jobConfig);
+ log.info("Response for job {}, {}", jobIdAndName, response);
+
+ jobModel.setJobId(jobIdAndName);
+ jobStatus.setReason("Successfully Submitted");
+ jobModel.setJobStatuses(Arrays.asList(jobStatus ));
+ jobStatus.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
+ taskContext.getParentProcessContext().setJobModel(jobModel);
+
+ GFacUtils.saveJobModel(processContext, jobModel);
+ GFacUtils.saveJobStatus(processContext, jobModel);
+ taskStatus.setReason("Successfully submitted job to Aurora");
+ } catch (Exception e) {
+ String msg = "Error occurred while submitting the job";
+ log.error(msg, e);
+ taskStatus.setState(TaskState.FAILED);
+ taskStatus.setReason(msg);
+ taskStatus.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
+ ErrorModel errorModel = new ErrorModel();
+ errorModel.setActualErrorMessage(e.getMessage());
+ errorModel.setUserFriendlyMessage(msg);
+ taskContext.getTaskModel().setTaskErrors(Arrays.asList(errorModel));
+ }
+
+ taskContext.setTaskStatus(taskStatus);
+ try {
+ GFacUtils.saveAndPublishTaskStatus(taskContext);
+ } catch (GFacException e) {
+ log.error("Error while saving task status", e);
+ }
+ return taskStatus;
+ }
+
+ @Override
+ public TaskStatus recover(TaskContext taskContext) {
+ return null;
+ }
+
+ @Override
+ public TaskTypes getType() {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/21b1923d/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/monitor/cloud/AuroraJobMonitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/monitor/cloud/AuroraJobMonitor.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/monitor/cloud/AuroraJobMonitor.java
new file mode 100644
index 0000000..5fe9dd8
--- /dev/null
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/monitor/cloud/AuroraJobMonitor.java
@@ -0,0 +1,247 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.airavata.gfac.monitor.cloud;
+
+import org.apache.airavata.cloud.aurora.client.AuroraThriftClient;
+import org.apache.airavata.cloud.aurora.client.bean.JobDetailsResponseBean;
+import org.apache.airavata.cloud.aurora.client.bean.JobKeyBean;
+import org.apache.airavata.cloud.aurora.client.sdk.ScheduledTask;
+import org.apache.airavata.common.utils.AiravataUtils;
+import org.apache.airavata.gfac.core.GFacException;
+import org.apache.airavata.gfac.core.GFacThreadPoolExecutor;
+import org.apache.airavata.gfac.core.GFacUtils;
+import org.apache.airavata.gfac.core.context.ProcessContext;
+import org.apache.airavata.gfac.core.context.TaskContext;
+import org.apache.airavata.gfac.core.monitor.JobMonitor;
+import org.apache.airavata.gfac.impl.AuroraUtils;
+import org.apache.airavata.gfac.impl.GFacWorker;
+import org.apache.airavata.model.job.JobModel;
+import org.apache.airavata.model.status.JobState;
+import org.apache.airavata.model.status.JobStatus;
+import org.apache.airavata.model.status.ProcessState;
+import org.apache.airavata.model.status.ProcessStatus;
+import org.apache.airavata.model.status.TaskState;
+import org.apache.airavata.model.status.TaskStatus;
+import org.apache.airavata.registry.cpi.ExperimentCatalog;
+import org.apache.airavata.registry.cpi.ExperimentCatalogModelType;
+import org.apache.airavata.registry.cpi.RegistryException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class AuroraJobMonitor implements JobMonitor, Runnable {
+ private static final Logger log = LoggerFactory.getLogger(AuroraJobMonitor.class);
+
+
+
+ private static AuroraJobMonitor auroraJobMonitor;
+ private Timer timer;
+ private Map<String,TaskContext> jobMonitoringMap;
+ private AuroraJobMonitor(){
+ jobMonitoringMap = new ConcurrentHashMap<>();
+ timer = new Timer("Aurora status poll timer");
+
+ }
+
+ public static AuroraJobMonitor getInstance(){
+ if (auroraJobMonitor == null) {
+ synchronized (AuroraJobMonitor.class){
+ if (auroraJobMonitor == null) {
+ auroraJobMonitor = new AuroraJobMonitor();
+ }
+ }
+ }
+ return auroraJobMonitor;
+ }
+ @Override
+ public void run() {
+ AuroraTimer task = null;
+ try {
+ task = new AuroraTimer();
+ timer.schedule(task, 5000, 2000);
+ } catch (Exception e) {
+ log.error("Error couldn't run Aurora status poll timer task");
+ }
+ }
+
+ @Override
+ public void monitor(String jobId, TaskContext taskContext) {
+ jobMonitoringMap.put(jobId, taskContext);
+ log.info("Added JobId : {} to Aurora Job Monitoring map", jobId);
+ taskContext.getParentProcessContext().setPauseTaskExecution(true);
+
+ }
+
+ @Override
+ public void stopMonitor(String jobId, boolean runOutFlow) {
+ jobMonitoringMap.remove(jobId);
+ }
+
+ @Override
+ public boolean isMonitoring(String jobId) {
+ return jobMonitoringMap.get(jobId) != null;
+ }
+
+ @Override
+ public void canceledJob(String jobId) {
+ throw new IllegalStateException("Method not yet implemented");
+ }
+
+ class AuroraTimer extends TimerTask {
+
+ AuroraThriftClient client;
+ public AuroraTimer() throws Exception {
+ client = AuroraThriftClient.getAuroraThriftClient(AuroraUtils.AURORA_SCHEDULER_PROP_FILE);
+
+ }
+
+
+ @Override
+
+ public void run() {
+ JobKeyBean jobKeyBean = new JobKeyBean(AuroraUtils.ENVIRONMENT, AuroraUtils.ROLE, "dummy");
+ Iterator<Map.Entry<String, TaskContext>> iterator = jobMonitoringMap.entrySet().iterator();
+ while (iterator.hasNext()) {
+ Map.Entry<String, TaskContext> currentEntry = iterator.next();
+ try {
+ jobKeyBean.setName(currentEntry.getKey());
+ JobDetailsResponseBean jobDetailsResponseBean = client.getJobDetails(jobKeyBean);
+ List<ScheduledTask> tasks = jobDetailsResponseBean.getTasks();
+ switch (tasks.get(0).getStatus()) {
+ case FINISHED:
+ iterator.remove();
+ processJob(currentEntry.getKey(), currentEntry.getValue(), JobState.COMPLETE);
+ break;
+ case FAILED:
+ iterator.remove();
+ processJob(currentEntry.getKey(), currentEntry.getValue(), JobState.FAILED);
+ break;
+ case RUNNING:
+ updateStatus(currentEntry.getKey(), currentEntry.getValue(), JobState.ACTIVE);
+ break;
+ default:
+ log.info("Job {} is in {} state", currentEntry.getKey(), tasks.get(0).getStatus().name());
+ break;
+ }
+ } catch (Exception e) {
+ log.error("Error while getting response for job : {}", currentEntry.getKey());
+
+ }
+ }
+ }
+
+ private void updateStatus(String jobKey, TaskContext taskContext, JobState jobState) {
+ ProcessContext pc = taskContext.getParentProcessContext();
+ ExperimentCatalog experimentCatalog = pc.getExperimentCatalog();
+ List<Object> objects = null;
+ try {
+ objects = experimentCatalog.get(ExperimentCatalogModelType.JOB_STATUS, taskContext.getTaskId(), jobKey);
+ } catch (RegistryException e) {
+ log.error("Error while getting job statuses for job : {} , task : {}, process : {}", jobKey,
+ taskContext.getTaskId(), pc.getProcessId());
+ }
+ List<JobState> jobStatuses = objects.stream()
+ .map(o -> ((JobStatus) o).getJobState())
+ .collect(Collectors.toList());
+ if (!jobStatuses.contains(jobState)) {
+ JobStatus jobStatus = new JobStatus(jobState);
+ jobStatus.setReason("Aurora return " + jobState.name());
+ jobStatus.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
+ JobModel jobModel = pc.getJobModel();
+ jobModel.setJobStatuses(Arrays.asList(jobStatus));
+ try {
+ GFacUtils.saveJobStatus(pc, jobModel);
+ } catch (GFacException e) {
+ log.error("Error while saving job status {}, job : {}, task :{}, process:{} exp:{}",
+ jobState.name(), jobKey, taskContext.getTaskId(), pc.getProcessId(), pc.getExperimentId());
+ }
+ }
+ }
+
+ private void processJob(String jobKey, TaskContext taskContext, JobState jobState) {
+ JobStatus jobStatus = new JobStatus();
+ jobStatus.setJobState(jobState);
+ if (jobState == JobState.COMPLETE) {
+ jobStatus.setReason("Aurora Job completed");
+ } else if (jobState == JobState.FAILED) {
+ jobStatus.setReason("Aurora Job Failed");
+ }
+ ProcessContext pc = taskContext.getParentProcessContext();
+ JobModel jobModel = pc.getJobModel();
+ jobStatus.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
+ jobModel.setJobStatuses(Arrays.asList(jobStatus));
+ try {
+ GFacUtils.saveJobStatus(pc, jobModel);
+ } catch (GFacException e) {
+ log.error("Error while saving job status for job : {} ", jobKey);
+ }
+
+ TaskStatus taskStatus = new TaskStatus(TaskState.COMPLETED);
+ taskStatus.setReason("Job monitoring completed with final state: " + TaskState.COMPLETED.name());
+ taskStatus.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
+ taskContext.setTaskStatus(taskStatus);
+ try {
+ GFacUtils.saveAndPublishTaskStatus(taskContext);
+ } catch (GFacException e) {
+ log.error("Error while saving task status for exp : {} , process : {} , task : {} , job : {}",
+ taskContext.getExperimentId(), taskContext.getProcessId(), taskContext.getTaskId(), jobKey);
+ }
+
+ if (pc.isCancel()) {
+ ProcessStatus processStatus = new ProcessStatus(ProcessState.CANCELLING);
+ processStatus.setReason("Process has been cancelled");
+ pc.setProcessStatus(processStatus);
+ try {
+ GFacUtils.saveAndPublishProcessStatus(pc);
+ } catch (GFacException e) {
+ log.error("Error while cancelling process, exp : {}, process : {}", pc.getExperimentId(), pc.getProcessId());
+ }
+ }
+
+ try {
+ GFacThreadPoolExecutor.getCachedThreadPool().execute(new GFacWorker(pc));
+ } catch (GFacException e) {
+ log.error("Error while running output tasks for exp : {} , process : {}", taskContext.getExperimentId(), pc.getProcessId());
+
+ ProcessStatus processStatus = new ProcessStatus(ProcessState.FAILED);
+ processStatus.setReason("Failed to run output tasks");
+ processStatus.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
+ pc.setProcessStatus(processStatus);
+ try {
+ GFacUtils.saveAndPublishProcessStatus(pc);
+ } catch (GFacException ex) {
+ log.error("Error while updating process status to FAILED, exp : {}, process : {}", pc.getExperimentId(), pc.getProcessId());
+ }
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/21b1923d/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java
index 74bd2db..83c9273 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java
@@ -262,6 +262,11 @@ public class OrchestratorUtils {
if (sshJobSubmission != null) {
return sshJobSubmission.getSecurityProtocol();
}
+ } else if (submissionProtocol == JobSubmissionProtocol.CLOUD) {
+ CloudJobSubmission cloudJobSubmission = getCloudJobSubmission(context, jobSubmissionInterface.getJobSubmissionInterfaceId());
+ if (cloudJobSubmission != null) {
+ return cloudJobSubmission.getSecurityProtocol();
+ }
}
} catch (RegistryException e) {
logger.error("Error occurred while retrieving security protocol", e);
@@ -302,6 +307,17 @@ public class OrchestratorUtils {
}
}
+ public static CloudJobSubmission getCloudJobSubmission(OrchestratorContext context, String submissionId) throws RegistryException {
+ try {
+ AppCatalog appCatalog = context.getRegistry().getAppCatalog();
+ return appCatalog.getComputeResource().getCloudJobSubmission(submissionId);
+ } catch (Exception e) {
+ String errorMsg = "Error while retrieving SSH job submission with submission id : " + submissionId;
+ logger.error(errorMsg, e);
+ throw new RegistryException(errorMsg, e);
+ }
+ }
+
public static SCPDataMovement getSCPDataMovement(OrchestratorContext context, String dataMoveId) throws RegistryException {
try {
AppCatalog appCatalog = context.getRegistry().getAppCatalog();
http://git-wip-us.apache.org/repos/asf/airavata/blob/21b1923d/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
index dec9b2c..b97e79a 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
@@ -481,6 +481,8 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{
monitorMode = MonitorMode.FORK;
} else if(jobSubmissionProtocol == JobSubmissionProtocol.LOCAL){
monitorMode = MonitorMode.LOCAL;
+ } else if (jobSubmissionProtocol == JobSubmissionProtocol.CLOUD) {
+ monitorMode = MonitorMode.CLOUD_JOB_MONITOR;
}else {
logger.error("expId : {}, processId : {} :- Unsupported Job submission protocol {}.",
processModel.getExperimentId(), processModel.getProcessId(), jobSubmissionProtocol.name());
@@ -507,7 +509,8 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{
submissionTaskIds.add(taskModel.getTaskId());
// create monitor task for this Email based monitor mode job
- if (monitorMode == MonitorMode.JOB_EMAIL_NOTIFICATION_MONITOR) {
+ if (monitorMode == MonitorMode.JOB_EMAIL_NOTIFICATION_MONITOR
+ || monitorMode == MonitorMode.CLOUD_JOB_MONITOR) {
TaskModel monitorTaskModel = new TaskModel();
monitorTaskModel.setParentProcessId(processModel.getProcessId());
monitorTaskModel.setCreationTime(new Date().getTime());
http://git-wip-us.apache.org/repos/asf/airavata/blob/21b1923d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 71fc9ba..81b258f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -626,7 +626,7 @@
<module>modules/orchestrator</module>
<module>modules/monitoring</module>
<module>modules/user-profile</module>
- <!--<module>modules/cloud</module>-->
+ <module>modules/cloud</module>
<module>modules/server</module>
<module>modules/workflow</module>
<module>modules/test-suite</module>
[2/6] airavata git commit: Merge pull request #2 from apache/develop
Posted by sh...@apache.org.
Merge pull request #2 from apache/develop
Merge airavata-develop
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/b66fa88f
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/b66fa88f
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/b66fa88f
Branch: refs/heads/develop
Commit: b66fa88fff988159e72617d34a97e1bd0e665600
Parents: 7dd8516 325d163
Author: Gourav Shenoy <sh...@gmail.com>
Authored: Fri Oct 28 12:46:17 2016 -0400
Committer: GitHub <no...@github.com>
Committed: Fri Oct 28 12:46:17 2016 -0400
----------------------------------------------------------------------
.../server/handler/AiravataServerHandler.java | 25 +
.../java/org/apache/airavata/api/Airavata.java | 9663 ++++++-----
.../main/resources/lib/airavata/Airavata.cpp | 6276 ++++---
.../src/main/resources/lib/airavata/Airavata.h | 533 +
.../lib/airavata/Airavata_server.skeleton.cpp | 77 +
.../resources/lib/airavata/airavata_api_types.h | 1 +
.../airavata/compute_resource_model_types.cpp | 4 +-
.../lib/airavata/compute_resource_model_types.h | 9 +-
.../airavata/credential_summary_constants.cpp | 34 +
.../lib/airavata/credential_summary_constants.h | 41 +
.../lib/airavata/credential_summary_types.cpp | 234 +
.../lib/airavata/credential_summary_types.h | 122 +
.../user_resource_profile_model_types.cpp | 22 +
.../user_resource_profile_model_types.h | 12 +-
.../resources/lib/Airavata/API/Airavata.php | 2118 ++-
.../Model/AppCatalog/ComputeResource/Types.php | 18 +-
.../Model/AppCatalog/CredetialSummary/Types.php | 210 +
.../AppCatalog/UserResourceProfile/Types.php | 788 +
.../Airavata/Model/Credential/Store/Types.php | 1029 ++
.../lib/apache/airavata/api/Airavata-remote | 21 +
.../lib/apache/airavata/api/Airavata.py | 1543 +-
.../resources/lib/apache/airavata/api/ttypes.py | 1 +
.../model/appcatalog/computeresource/ttypes.py | 27 +-
.../appcatalog/credentialsummary/__init__.py | 1 +
.../appcatalog/credentialsummary/constants.py | 11 +
.../appcatalog/credentialsummary/ttypes.py | 154 +
.../appcatalog/userresourceprofile/__init__.py | 1 +
.../appcatalog/userresourceprofile/constants.py | 11 +
.../appcatalog/userresourceprofile/ttypes.py | 538 +
.../main/resources/samples/AiravataClient.py | 306 +
.../resources/samples/testAiravataClient.py | 32 +
.../appcatalog/computeresource/MonitorMode.java | 17 +-
.../credentialsummary/CredentialSummary.java | 127 +-
.../UserResourceProfile.java | 116 +-
apache-license-header.txt | 17 +
.../cloud/ansible-playbooks/group_vars/all.yml | 20 +
.../ansible-playbooks/roles/ec2/tasks/main.yml | 20 +
.../roles/ec2/vars/aws-credential.yml | 20 +
.../ansible-playbooks/roles/ec2/vars/main.yml | 20 +
.../ansible-playbooks/roles/java/tasks/main.yml | 20 +
.../ansible-playbooks/roles/java/vars/main.yml | 20 +
.../roles/mesos-master/tasks/backup_main.yml | 20 +
.../roles/mesos-master/tasks/main.yml | 20 +
.../roles/mesos-slave/tasks/main.yml | 20 +
.../roles/openstack/tasks/main.yml | 20 +
.../roles/openstack/vars/main.yml | 20 +
.../openstack/vars/openstack-credentials.yml | 20 +
.../ansible-playbooks/roles/test/tasks/main.yml | 20 +
.../roles/zookeeper/handlers/main.yml | 20 +
.../roles/zookeeper/tasks/main.yml | 20 +
.../roles/zookeeper/vars/main.yml | 20 +
modules/cloud/ansible-playbooks/site.yml | 20 +
modules/cloud/aurora-client/README.md | 49 +
modules/cloud/aurora-client/pom.xml | 76 +
.../client/AuroraSchedulerClientFactory.java | 69 +
.../cloud/aurora/client/AuroraThriftClient.java | 216 +
.../aurora/client/bean/GetJobsResponseBean.java | 53 +
.../cloud/aurora/client/bean/IdentityBean.java | 58 +
.../cloud/aurora/client/bean/JobConfigBean.java | 236 +
.../client/bean/JobDetailsResponseBean.java | 69 +
.../cloud/aurora/client/bean/JobKeyBean.java | 104 +
.../client/bean/PendingJobReasonBean.java | 70 +
.../cloud/aurora/client/bean/ProcessBean.java | 193 +
.../cloud/aurora/client/bean/ResourceBean.java | 104 +
.../cloud/aurora/client/bean/ResponseBean.java | 79 +
.../aurora/client/bean/ServerInfoBean.java | 87 +
.../aurora/client/bean/TaskConfigBean.java | 136 +
.../cloud/aurora/client/sdk/AppcImage.java | 549 +
.../cloud/aurora/client/sdk/AssignedTask.java | 1084 ++
.../cloud/aurora/client/sdk/Attribute.java | 574 +
.../cloud/aurora/client/sdk/AuroraAdmin.java | 14400 +++++++++++++++++
.../client/sdk/AuroraSchedulerManager.java | 13414 +++++++++++++++
.../cloud/aurora/client/sdk/ConfigGroup.java | 581 +
.../cloud/aurora/client/sdk/ConfigRewrite.java | 389 +
.../cloud/aurora/client/sdk/ConfigSummary.java | 581 +
.../aurora/client/sdk/ConfigSummaryResult.java | 422 +
.../cloud/aurora/client/sdk/Constraint.java | 539 +
.../cloud/aurora/client/sdk/Container.java | 392 +
.../aurora/client/sdk/CronCollisionPolicy.java | 80 +
.../aurora/client/sdk/DockerContainer.java | 606 +
.../cloud/aurora/client/sdk/DockerImage.java | 549 +
.../aurora/client/sdk/DockerParameter.java | 549 +
.../aurora/client/sdk/DrainHostsResult.java | 471 +
.../aurora/client/sdk/EndMaintenanceResult.java | 471 +
.../cloud/aurora/client/sdk/ExecutorConfig.java | 546 +
.../sdk/ExplicitReconciliationSettings.java | 412 +
.../client/sdk/GetJobUpdateDetailsResult.java | 584 +
.../client/sdk/GetJobUpdateDiffResult.java | 996 ++
.../client/sdk/GetJobUpdateSummariesResult.java | 474 +
.../cloud/aurora/client/sdk/GetJobsResult.java | 471 +
.../client/sdk/GetPendingReasonResult.java | 471 +
.../cloud/aurora/client/sdk/GetQuotaResult.java | 931 ++
.../aurora/client/sdk/GetTierConfigResult.java | 603 +
.../cloud/aurora/client/sdk/HostAttributes.java | 810 +
.../cloud/aurora/client/sdk/HostStatus.java | 538 +
.../airavata/cloud/aurora/client/sdk/Hosts.java | 466 +
.../cloud/aurora/client/sdk/Identity.java | 417 +
.../airavata/cloud/aurora/client/sdk/Image.java | 392 +
.../client/sdk/InstanceConfigRewrite.java | 678 +
.../cloud/aurora/client/sdk/InstanceKey.java | 552 +
.../aurora/client/sdk/InstanceTaskConfig.java | 608 +
.../aurora/client/sdk/JobConfigRewrite.java | 556 +
.../aurora/client/sdk/JobConfiguration.java | 1049 ++
.../client/sdk/JobInstanceUpdateEvent.java | 666 +
.../cloud/aurora/client/sdk/JobKey.java | 666 +
.../cloud/aurora/client/sdk/JobStats.java | 760 +
.../cloud/aurora/client/sdk/JobSummary.java | 649 +
.../aurora/client/sdk/JobSummaryResult.java | 471 +
.../cloud/aurora/client/sdk/JobUpdate.java | 559 +
.../aurora/client/sdk/JobUpdateAction.java | 101 +
.../aurora/client/sdk/JobUpdateDetails.java | 776 +
.../cloud/aurora/client/sdk/JobUpdateEvent.java | 798 +
.../client/sdk/JobUpdateInstructions.java | 730 +
.../cloud/aurora/client/sdk/JobUpdateKey.java | 554 +
.../aurora/client/sdk/JobUpdatePulseStatus.java | 74 +
.../cloud/aurora/client/sdk/JobUpdateQuery.java | 1189 ++
.../aurora/client/sdk/JobUpdateRequest.java | 848 +
.../aurora/client/sdk/JobUpdateSettings.java | 1285 ++
.../cloud/aurora/client/sdk/JobUpdateState.java | 666 +
.../aurora/client/sdk/JobUpdateStatus.java | 129 +
.../aurora/client/sdk/JobUpdateSummary.java | 850 +
.../aurora/client/sdk/LimitConstraint.java | 419 +
.../aurora/client/sdk/ListBackupsResult.java | 466 +
.../airavata/cloud/aurora/client/sdk/Lock.java | 903 ++
.../cloud/aurora/client/sdk/LockKey.java | 331 +
.../aurora/client/sdk/MaintenanceMode.java | 70 +
.../client/sdk/MaintenanceStatusResult.java | 471 +
.../cloud/aurora/client/sdk/MesosContainer.java | 432 +
.../aurora/client/sdk/MesosFetcherURI.java | 661 +
.../cloud/aurora/client/sdk/Metadata.java | 525 +
.../airavata/cloud/aurora/client/sdk/Mode.java | 73 +
.../cloud/aurora/client/sdk/Package.java | 625 +
.../cloud/aurora/client/sdk/PendingReason.java | 522 +
.../aurora/client/sdk/PopulateJobResult.java | 422 +
.../aurora/client/sdk/PulseJobUpdateResult.java | 436 +
.../aurora/client/sdk/QueryRecoveryResult.java | 471 +
.../airavata/cloud/aurora/client/sdk/Range.java | 517 +
.../aurora/client/sdk/ReadOnlyScheduler.java | 11291 +++++++++++++
.../cloud/aurora/client/sdk/Resource.java | 561 +
.../aurora/client/sdk/ResourceAggregate.java | 820 +
.../cloud/aurora/client/sdk/Response.java | 840 +
.../cloud/aurora/client/sdk/ResponseCode.java | 85 +
.../cloud/aurora/client/sdk/ResponseDetail.java | 417 +
.../cloud/aurora/client/sdk/Result.java | 1487 ++
.../client/sdk/RewriteConfigsRequest.java | 471 +
.../cloud/aurora/client/sdk/RoleSummary.java | 619 +
.../aurora/client/sdk/RoleSummaryResult.java | 471 +
.../cloud/aurora/client/sdk/ScheduleStatus.java | 143 +
.../aurora/client/sdk/ScheduleStatusResult.java | 471 +
.../cloud/aurora/client/sdk/ScheduledTask.java | 973 ++
.../cloud/aurora/client/sdk/ServerInfo.java | 537 +
.../aurora/client/sdk/StartJobUpdateResult.java | 562 +
.../client/sdk/StartMaintenanceResult.java | 471 +
.../cloud/aurora/client/sdk/TaskConfig.java | 2769 ++++
.../cloud/aurora/client/sdk/TaskConstraint.java | 392 +
.../cloud/aurora/client/sdk/TaskEvent.java | 794 +
.../cloud/aurora/client/sdk/TaskQuery.java | 1611 ++
.../cloud/aurora/client/sdk/TierConfig.java | 601 +
.../aurora/client/sdk/ValueConstraint.java | 585 +
.../cloud/aurora/client/sdk/Volume.java | 674 +
.../aurora/client/sdk/aurora_apiConstants.java | 124 +
.../cloud/aurora/sample/AuroraClientSample.java | 157 +
.../aurora/util/AuroraThriftClientUtil.java | 383 +
.../airavata/cloud/aurora/util/Constants.java | 44 +
.../cloud/aurora/util/ResponseCodeEnum.java | 95 +
.../cloud/aurora/util/ResponseResultType.java | 47 +
.../src/main/resources/aurora-api.thrift | 1238 ++
.../main/resources/aurora-scheduler.properties | 9 +
.../resources/executor-config-template.json | 48 +
modules/cloud/cloud-provisioning/pom.xml | 31 +-
.../airavata/cloud/intf/CloudInterface.java | 4 +-
.../cloud/intf/impl/OpenstackIntfImpl.java | 4 +-
.../cloud/openstack/OS4JClientProvider.java | 4 +-
.../apache/airavata/cloud/util/CloudRef.java | 4 +-
.../apache/airavata/cloud/util/Constants.java | 6 +-
.../org/apache/airavata/cloud/util/IPType.java | 4 +-
.../airavata/cloud/util/OpenstackIntfUtil.java | 4 +-
.../airavata/cloud/test/CloudIntfTest.java | 4 +-
modules/cloud/pom.xml | 57 +-
.../org/apache/airavata/cloud/CloudMain.java | 3 +-
.../store/store/impl/db/CredentialsDAO.java | 4 +-
.../cpi/SSHSummaryTest/SSHSummaryTest.java | 173 +
.../store/store/impl/db/CredentialsDAOTest.java | 4 +-
.../gfac/core/DataStagingException.java | 43 +
.../gfac/core/cluster/RemoteCluster.java | 33 +-
.../gfac/core/cluster/SessionConsumer.java | 28 +
.../airavata/gfac/impl/BESRemoteCluster.java | 9 +-
.../airavata/gfac/impl/HPCRemoteCluster.java | 37 +-
.../airavata/gfac/impl/LocalRemoteCluster.java | 26 +-
.../org/apache/airavata/gfac/impl/SSHUtils.java | 198 +-
.../airavata/gfac/impl/task/ArchiveTask.java | 8 +-
.../gfac/impl/task/BESJobSubmissionTask.java | 5 +-
.../airavata/gfac/impl/task/DataStageTask.java | 13 +-
.../gfac/impl/task/EnvironmentSetupTask.java | 4 +-
.../gfac/impl/task/SCPDataStageTask.java | 31 +-
.../gfac/impl/task/utils/StreamData.java | 37 +-
.../catalog/impl/UsrResourceProfileImpl.java | 4 +-
.../UserStoragePreferenceResource.java | 4 +-
.../util/AppCatalogThriftConversion.java | 5 +
.../app/catalog/UserResourceProfileTest.java | 31 +-
.../airavata-apis/airavata_api.thrift | 26 +
.../compute_resource_model.thrift | 1 +
.../credential_summary.thrift | 7 +-
.../user_resource_profile_model.thrift | 6 +-
204 files changed, 115586 insertions(+), 7464 deletions(-)
----------------------------------------------------------------------