You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by sh...@apache.org on 2015/04/08 17:23:44 UTC
[1/7] airavata git commit: Add new gfac email monitor module to gfac
monitor module and Intorduced gfac-hpc-monitor module to keep all hoc-monitor
code
Repository: airavata
Updated Branches:
refs/heads/emailMonitoring 95fae20b1 -> 6ffc3ee70
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/Endpoint.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/Endpoint.json b/modules/gfac/gfac-monitor/src/main/resources/schema/Endpoint.json
deleted file mode 100644
index b75b02a..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/Endpoint.json
+++ /dev/null
@@ -1,147 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Endpoint.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
- "properties": {
- "URL": {
- "type": "string",
- "description": "Network location of the endpoint"
- },
- "Capability": {
- "type": "array",
- "description": "Capability_t (open enumeration)",
- "items": {
- "type": "string"
- }
- },
- "Technology": {
- "type": "string",
- "description": "EndpointTechnology_t"
- },
- "InterfaceName": {
- "type": "string",
- "description": "InterfaceName_t"
- },
- "InterfaceVersion": {
- "type": "string",
- "description": "The version of the primary interface protocol (free format)"
- },
- "InterfaceExtension": {
- "type": "array",
- "description": "URIs identifying supported extensions to the interface protocol",
- "items": {
- "type": "string"
- }
- },
- "WSDL": {
- "type": "array",
- "description": "URLs of WSDL document(s) describing the interface",
- "items": {
- "type": "string"
- }
- },
- "SupportedProfile": {
- "type": "array",
- "description": "URI(s) identifying supported profiles for the Endpoint",
- "items": {
- "type": "string"
- }
- },
- "Semantics": {
- "type": "array",
- "description": "URL(s) of documents providing human-readable descriptions of the semantics of the Endpoint",
- "items": {
- "type": "string"
- }
- },
- "Implementor": {
- "type": "string",
- "description": "The name of the main organization implementing the Endpoint"
- },
- "ImplementationName": {
- "type": "string",
- "description": "The name of the implementation of the Endpoint"
- },
- "ImplementationVersion": {
- "type": "string",
- "description": "The version of the implementation of the Endpoint"
- },
- "QualityLevel": {
- "type": "string",
- "description": "QualityLevel_t",
- "enum": ["development","pre-production","production","testing"]
- },
- "HealthState": {
- "type": "string",
- "description": "The operational status of the Endpoint",
- "enum": ["critical","ok","other","unknown","warning"]
- },
- "HealthStateInfo": {
- "type": "string",
- "description": "A human-readable explanation of the HealthState of this Endpoint"
- },
- "ServingState": {
- "type": "string",
- "description": "If the endpoint is accepting and serving requests",
- "enum": ["closed","draining","production","queueing"]
- },
- "StartTime": {
- "type": "string",
- "description": "The start time of the Service associated with this Endpoint (DateTime_t)"
- },
- "IssuerCA": {
- "type": "string",
- "description": "The DN of the CA issuing the certificate presented by this Endpoint"
- },
- "TrustedCA": {
- "type": "array",
- "description": "DN(s) of CAs trusted by this Endpoint",
- "items": {
- "type": "string"
- }
- },
- "DowntimeAnnounce": {
- "type": "string",
- "description": "When the next scheduled downtime was announced (DateTime_t)"
- },
- "DowntimeStart": {
- "type": "string",
- "description": "When the next scheduled downtime will start (DateTime_t)"
- },
- "DowntimeEnd": {
- "type": "string",
- "description": "When the next scheduled downtime will end (DateTime_t)"
- },
- "DowntimeInfo": {
- "type": "string",
- "description": "Human-readable of the next scheduled downtime"
- },
- "ServiceID": {
- "type": "string",
- "description": "The ID of the Service associated with this Endpoint"
- },
- "ShareID": {
- "type": "array",
- "description": "The IDs of the Shares accessible from this Endpoint",
- "items": {
- "type": "string"
- }
- },
- "AccessPolicyID": {
- "type": "array",
- "description": "IDs of AccessPolicies associated with this Endpoint",
- "items": {
- "type": "string"
- }
- },
- "ActivityID": {
- "type": "array",
- "description": "IDs of Activities being managed through this Endpoint",
- "items": {
- "type": "string"
- }
- }
- },
- "required": ["InterfaceName","QualityLevel","HealthState","ServingState","ServiceID"]
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/Entity.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/Entity.json b/modules/gfac/gfac-monitor/src/main/resources/schema/Entity.json
deleted file mode 100644
index 5d1ae46..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/Entity.json
+++ /dev/null
@@ -1,35 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json",
- "type": "object",
- "properties": {
- "CreationTime": {
- "type": "string",
- "description": "The creation time of this entity in the format: CCYY-MM-DDThh:mm:ss[Z|(+|-)hh:mm]"
- },
- "Validity": {
- "type": "integer",
- "description": "The number of seconds after CreationTime that this entity should be considered relevant"
- },
- "ID": {
- "type": "string",
- "description": "A globally unique identifier for this entity"
- },
- "Name": {
- "type": "string",
- "description": "A human-readable name"
- },
- "OtherInfo": {
- "type": "array",
- "description": "Placeholder for information that does not fit in any other attribute",
- "items": {
- "type": "string"
- }
- },
- "Extension": {
- "type": "object",
- "description": "Key/value pairs enabling the association of extra information not captured by the model"
- }
- },
- "required": ["ID"]
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/ExecutionEnvironment.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/ExecutionEnvironment.json b/modules/gfac/gfac-monitor/src/main/resources/schema/ExecutionEnvironment.json
deleted file mode 100644
index 77bf876..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/ExecutionEnvironment.json
+++ /dev/null
@@ -1,115 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ExecutionEnvironment.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Resource.json"}],
- "properties": {
- "Platform": {
- "type": "string",
- "description": "The platform architecture - Platform_t (open enumeration)"
- },
- "VirtualMachine": {
- "type": "boolean",
- "description": "True if the ExecutionEnvironment is a virtual machine"
- },
- "TotalInstances": {
- "type": "integer",
- "description": "The total number of ExecutionEnvironment instances"
- },
- "UsedInstances": {
- "type": "integer",
- "description": "The number of ExecutionEnvironment instances in use"
- },
- "UnavailableInstances": {
- "type": "integer",
- "description": "The number of ExecutionEnvironment instances that are unavailable"
- },
- "PhysicalCPUs": {
- "type": "integer",
- "description": "The number of physical CPUs in one ExecutionEnvironment instance"
- },
- "LogicalCPUs": {
- "type": "integer",
- "description": "The number of logical CPUs in one ExecutionEnvironment instance"
- },
- "CPUMultiplicity": {
- "type": "string",
- "description": "Information about the CPUs and cores in an execution environment",
- "enum": ["multicpu-multicore","multicpu-singlecore","singlecpu-multicore","singlecpu-singlecore"]
- },
- "CPUVendor": {
- "type": "string",
- "description": "The name of the manufacturer of the CPU"
- },
- "CPUModel": {
- "type": "string",
- "description": "The model of the CPU, as defined by the vendor"
- },
- "CPUVersion": {
- "type": "string",
- "description": "The specific version name of the CPU, as defined by the vendor"
- },
- "CPUClockSpeed": {
- "type": "integer",
- "description": "The clock speed of the CPU (MHz)"
- },
- "CPUTimeScalingFactor": {
- "type": "float",
- "description": "The factor used by the ComputingManager to scale the CPU time limit"
- },
- "WallTimeScalingFactor": {
- "type": "float",
- "description": "The factor used by the ComputingManager to scale the wallclock time limit"
- },
- "MainMemorySize": {
- "type": "integer",
- "description": "The total amount of physical RAM in one ExecutionEnvironment instance (MB)"
- },
- "VirtualMemorySize": {
- "type": "integer",
- "description": "The total amount of virtual memory (RAM+swap) in one ExecutionEnvironment instance (MB)"
- },
- "OSFamily": {
- "type": "string",
- "description": "The general family of the operating system - OSFamily_t (open enumeration)"
- },
- "OSName": {
- "type": "string",
- "description": "The specific name of the operating system - OSName_t (open enumeration)"
- },
- "OSVersion": {
- "type": "string",
- "description": "The version of the operating system, as defined by the vendor"
- },
- "ConnectivityIn": {
- "type": "boolean",
- "description": "True if direct inbound network connectiity is available to a running job"
- },
- "ConnectivityOut": {
- "type": "boolean",
- "description": "True if direct outbound network connectiity is available to a running job"
- },
- "NetworkInfo": {
- "type": "array",
- "description": "The types of internal network connections between ExecutionEnvironments - NetworkInfo_t (open enumeration)",
- "items": {
- "type": "string"
- }
- },
- "ApplicationEnvironmentID": {
- "type": "array",
- "description": "ID(s) of ApplicationEnvironments available in this ExecutionEnvironment",
- "items": {
- "type": "string"
- }
- },
- "BenchmarkID": {
- "type": "array",
- "description": "ID(s) of Benchmarks associated with this ExecutionEnvironment",
- "items": {
- "type": "string"
- }
- }
- },
- "required": ["Platform","MainMemorySize","OSFamily","ConnectivityIn","ConnectivityOut"]
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/Glue2.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/Glue2.json b/modules/gfac/gfac-monitor/src/main/resources/schema/Glue2.json
deleted file mode 100644
index bb80505..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/Glue2.json
+++ /dev/null
@@ -1,246 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Glue2.json",
- "description": "A GLUE 2 document",
- "type": "object",
- "properties": {
- "Entity": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}]
- }
- },
- "Location": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Location.json"}]
- }
- },
- "Contact": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Contact.json"}]
- }
- },
- "Domain": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Domain.json"}]
- }
- },
- "AdminDomain": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/AdminDomain.json"}]
- }
- },
- "UserDomain": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/UserDomain.json"}]
- }
- },
- "Service": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Service.json"}]
- }
- },
- "Endpoint": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Endpoint.json"}]
- }
- },
- "Share": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Share.json"}]
- }
- },
- "Manager": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Manager.json"}]
- }
- },
- "Resource": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Resource.json"}]
- }
- },
- "Activity": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Activity.json"}]
- }
- },
- "Policy": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Policy.json"}]
- }
- },
- "AccessPolicy": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/AccessPolicy.json"}]
- }
- },
- "MappingPolicy": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/MappingPolicy.json"}]
- }
- },
- "ComputingService": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ComputingService.json"}]
- }
- },
- "ComputingEndpoint": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ComputingEndpoint.json"}]
- }
- },
- "ComputingShare": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ComputingShare.json"}]
- }
- },
- "ComputingManager": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ComputingManager.json"}]
- }
- },
- "Benchmark": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Benchmark.json"}]
- }
- },
- "ExecutionEnvironment": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ExecutionEnvironment.json"}]
- }
- },
- "ApplicationEnvironment": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ApplicationEnvironment.json"}]
- }
- },
- "ApplicationHandle": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ApplicationHandle.json"}]
- }
- },
- "ComputingActivity": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ComputingActivity.json"}]
- }
- },
- "ToStorageService": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ToStorageService.json"}]
- }
- },
- "StorageService": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageService.json"}]
- }
- },
- "StorageServiceCapacity": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageServiceCapacity.json"}]
- }
- },
- "StorageAccessProtocol": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageAccessProtocol.json"}]
- }
- },
- "StorageEndpoint": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageEndpoint.json"}]
- }
- },
- "StorageShare": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageShare.json"}]
- }
- },
- "StorageShareCapacity": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageShareCapacity.json"}]
- }
- },
- "StorageManager": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageManager.json"}]
- }
- },
- "DataStore": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/DataStore.json"}]
- }
- },
- "ToComputingService": {
- "type": "array",
- "items": {
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ToComputingService.json"}]
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/Location.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/Location.json b/modules/gfac/gfac-monitor/src/main/resources/schema/Location.json
deleted file mode 100644
index 8491cc0..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/Location.json
+++ /dev/null
@@ -1,47 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Location.json",
- "description": "A GLUE 2 Location",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
- "properties": {
- "Address": {
- "type": "string",
- "description": "A free format street address"
- },
- "Place": {
- "type": "string",
- "description": "Name of town/city"
- },
- "Country": {
- "type": "string",
- "description": "Name of country"
- },
- "PostalCode": {
- "type": "string",
- "description": "Postal code"
- },
- "Latitude": {
- "type": "number",
- "description": "Position north (positive) or south (negative) of the equator in degrees"
- },
- "Longitude": {
- "type": "number",
- "description": "Position east (positive) or west (negative) of the primary meridian in degrees"
- },
- "ServiceID": {
- "type": "array",
- "description": "The IDs of Services at this location",
- "items": {
- "type": "string"
- }
- },
- "DomainID": {
- "type": "array",
- "description": "The IDs of Domains at this location",
- "items": {
- "type": "string"
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/Manager.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/Manager.json b/modules/gfac/gfac-monitor/src/main/resources/schema/Manager.json
deleted file mode 100644
index d1df50a..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/Manager.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Manager.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
- "properties": {
- "ProductName": {
- "type": "string",
- "description": "The name of the software product which implements the Manager"
- },
- "ProductVersion": {
- "type": "string",
- "description": "The version of the software product which implements the Manager"
- },
- "ServiceID": {
- "type": "string",
- "description": "The ID of the Service this Share participates in"
- },
- "ResourceID": {
- "type": "array",
- "description": "ID(s) of Resources associated with this Share",
- "items": {
- "type": "string"
- }
- }
- },
- "required": ["ProductName","ServiceID","ResourceID"]
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/MappingPolicy.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/MappingPolicy.json b/modules/gfac/gfac-monitor/src/main/resources/schema/MappingPolicy.json
deleted file mode 100644
index 268844d..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/MappingPolicy.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/MappingPolicy.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Policy.json"}],
- "properties": {
- "ShareID": {
- "type": "string",
- "description": "The ID of the Share this MappingPolicy is for"
- }
- },
- "required": ["ShareID"]
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/Policy.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/Policy.json b/modules/gfac/gfac-monitor/src/main/resources/schema/Policy.json
deleted file mode 100644
index f936699..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/Policy.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Policy.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
- "properties": {
- "Scheme": {
- "type": "string",
- "description": "PolicyScheme_t (open enumeration)"
- },
- "Rule": {
- "type": "array",
- "description": "Policy rules",
- "items": {
- "type": "string"
- }
- },
- "UserDomainID": {
- "type": "array",
- "description": "The ID(s) of the UserDomains this Policy applies to",
- "items": {
- "type": "string"
- }
- }
- },
- "required": ["Scheme","Rule","UserDomainID"]
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/Resource.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/Resource.json b/modules/gfac/gfac-monitor/src/main/resources/schema/Resource.json
deleted file mode 100644
index 88d08ad..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/Resource.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Resource.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
- "properties": {
- "ManagerID": {
- "type": "string",
- "description": "The ID of the Manager for this Resource"
- },
- "ShareID": {
- "type": "array",
- "description": "The ID(s) of the Shares this Resource is part of",
- "items": {
- "type": "string"
- }
- },
- "ActivityID": {
- "type": "array",
- "description": "The ID(s) of Activities consuming from this Share",
- "items": {
- "type": "string"
- }
- }
- },
- "required": ["ManagerID"]
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/Service.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/Service.json b/modules/gfac/gfac-monitor/src/main/resources/schema/Service.json
deleted file mode 100644
index 4662407..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/Service.json
+++ /dev/null
@@ -1,75 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Service.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
- "properties": {
- "Capability": {
- "type": "array",
- "description": "Capability_t (open enumeration)",
- "items": {
- "type": "string"
- }
- },
- "Type": {
- "type": "string",
- "description": "ServiceType_t (open enumeration)"
- },
- "QualityLevel": {
- "type": "string",
- "description": "QualityLevel_t",
- "enum": ["development","pre-production","production","testing"]
- },
- "StatusInfo": {
- "type": "array",
- "description": "URLs of web pages providing additional information",
- "items": {
- "type": "string"
- }
- },
- "Complexity": {
- "type": "string",
- "description": "A human-readable description of the number of endpoint types, shares, and resources"
- },
- "EndpointID": {
- "type": "array",
- "description": "The IDs of Endpoints for this Service",
- "items": {
- "type": "string"
- }
- },
- "ShareID": {
- "type": "array",
- "description": "The IDs of the Shares offered by this Service",
- "items": {
- "type": "string"
- }
- },
- "ManagerID": {
- "type": "array",
- "description": "The IDs of the Managers of this Service",
- "items": {
- "type": "string"
- }
- },
- "ContactID": {
- "type": "array",
- "description": "The IDs of Contacts for this Service",
- "items": {
- "type": "string"
- }
- },
- "LocationID": {
- "type": "string",
- "description": "The ID of the primary Location of this Service"
- },
- "ServiceID": {
- "type": "array",
- "description": "The IDs of Services related to this Service",
- "items": {
- "type": "string"
- }
- }
- },
- "required": ["Type","QualityLevel"]
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/Share.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/Share.json b/modules/gfac/gfac-monitor/src/main/resources/schema/Share.json
deleted file mode 100644
index 258fc1b..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/Share.json
+++ /dev/null
@@ -1,45 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Share.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
- "properties": {
- "Description": {
- "type": "string",
- "description": "A human-readable description of the Share"
- },
- "EndpointID": {
- "type": "array",
- "description": "The ID(s) of the Endpoints that can be used to access this Share",
- "items": {
- "type": "string"
- }
- },
- "ResourceID": {
- "type": "array",
- "description": "The ID(s) of the Resources associated with this Share",
- "items": {
- "type": "string"
- }
- },
- "ServiceID": {
- "type": "string",
- "description": "The ID of the Service this Share participates in"
- },
- "ActivityID": {
- "type": "array",
- "description": "The ID(s) of Activities consuming from this Share",
- "items": {
- "type": "string"
- }
- },
- "MappingPolicyID": {
- "type": "array",
- "description": "ID(s) of MappingPolicies associated with this Share",
- "items": {
- "type": "string"
- }
- }
- },
- "required": ["ServiceID"]
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/StorageAccessProtocol.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/StorageAccessProtocol.json b/modules/gfac/gfac-monitor/src/main/resources/schema/StorageAccessProtocol.json
deleted file mode 100644
index 05a830b..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/StorageAccessProtocol.json
+++ /dev/null
@@ -1,32 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageAccessProtocol.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
- "properties": {
- "Type": {
- "type": "string",
- "description": "The type of the protocol - StorageAccessProtocol_t"
- },
- "Version": {
- "type": "string",
- "description": "The version of the protocol supported"
- },
- "MaxStreams": {
- "type": "integer",
- "description": "The maximum number of parallel network streams which can be usef for a single transfer"
- },
- "StorageServiceID": {
- "type": "string",
- "description": "The ID of the StorageService this protocol is available for"
- },
- "ToComputingServiceID": {
- "type": "array",
- "description": "The ID(s) ToComputingService objects that describe connectivity to ComputingServices",
- "items": {
- "type": "string"
- }
- }
- },
- "required": ["Type","Version","StorageServiceID"]
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/StorageEndpoint.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/StorageEndpoint.json b/modules/gfac/gfac-monitor/src/main/resources/schema/StorageEndpoint.json
deleted file mode 100644
index 38b27c4..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/StorageEndpoint.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageEndpoint.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Endpoint.json"}],
- "properties": {
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/StorageManager.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/StorageManager.json b/modules/gfac/gfac-monitor/src/main/resources/schema/StorageManager.json
deleted file mode 100644
index f3984f6..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/StorageManager.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageManager.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Share.json"}],
- "properties": {
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/StorageService.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/StorageService.json b/modules/gfac/gfac-monitor/src/main/resources/schema/StorageService.json
deleted file mode 100644
index a03d111..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/StorageService.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageService.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Service.json"}],
- "properties": {
- "StorageAccessProtocolID": {
- "type": "array",
- "description": "The ID(s) of the StorageAccessProtocols supported by this service",
- "items": {
- "type": "string"
- }
- },
- "StorageServiceCapacityID": {
- "type": "array",
- "description": "The ID(s) of the StorageServiceCapacities for this Service",
- "items": {
- "type": "string"
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/StorageServiceCapacity.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/StorageServiceCapacity.json b/modules/gfac/gfac-monitor/src/main/resources/schema/StorageServiceCapacity.json
deleted file mode 100644
index a25c204..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/StorageServiceCapacity.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageServiceCapacity.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
- "properties": {
- "Type": {
- "type": "string",
- "description": "The type of storage capacity - StorageCapacity_t"
- },
- "TotalSize": {
- "type": "integer",
- "description": "The total amount of storage of this type (GB)"
- },
- "FreeSize": {
- "type": "integer",
- "description": "The amount of currently available storage of this type (GB)"
- },
- "UsedSize": {
- "type": "integer",
- "description": "The amount storage of this type in use (GB)"
- },
- "ReservedSize": {
- "type": "integer",
- "description": "The amount storage of this type which is not in use, but has been reserved for use in use (GB)"
- },
- "StorageServiceID": {
- "type": "string",
- "description": "The ID of the StorageService this capacity describes"
- }
- },
- "required": ["Type","StorageServiceID"]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/StorageShare.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/StorageShare.json b/modules/gfac/gfac-monitor/src/main/resources/schema/StorageShare.json
deleted file mode 100644
index 9703118..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/StorageShare.json
+++ /dev/null
@@ -1,65 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageShare.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Share.json"}],
- "properties": {
- "ServingState": {
- "type": "string",
- "description": "How the Share is currently serving jobs",
- "enum": ["closed","draining","production","queueing"]
- },
- "Path": {
- "type": "string",
- "description": "A default namespace where files are logically placed when they are stored in this Share"
- },
- "AccessMode": {
- "type": "array",
- "description": "The type of access allowed for this share - AccessMode_t (undefined)",
- "items": {
- "type": "string"
- }
- },
- "SharingID": {
- "type": "string",
- "description": "A local identifier common to the set of StorageShares which use the same underling extents"
- },
- "AccessLatency": {
- "type": "string",
- "description": "The maximum latency category under normal operating conditions",
- "enum": ["nearline","offline","online"]
- },
- "RetentionPolicy": {
- "type": "string",
- "description": "The quality of data retention - RetentionPolicy_t"
- },
- "ExpirationMode": {
- "type": "array",
- "description": "Supported file lifetime modes",
- "items": {
- "type": "string",
- "enum": ["neverexpire","releasewhenexpired","warnwhenexpired"]
- }
- },
- "DefaultLifeTime": {
- "type": "integer",
- "description": "The default lifetime assigned to a new file"
- },
- "MaximumLifeTime": {
- "type": "integer",
- "description": "The maximum lifetime that can be requested for a file"
- },
- "Tag": {
- "type": "string",
- "description": "An identifier defined by a UserDomain"
- },
- "StorageShareCapacityID": {
- "type": "array",
- "description": "ID of the StorageShareCapacities associated with this share",
- "items": {
- "type": "string"
- }
- }
- },
- "required": ["ServingState","SharingID","AccessLatency"]
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/StorageShareCapacity.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/StorageShareCapacity.json b/modules/gfac/gfac-monitor/src/main/resources/schema/StorageShareCapacity.json
deleted file mode 100644
index f392c94..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/StorageShareCapacity.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageShareCapacity.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
- "properties": {
- "Type": {
- "type": "string",
- "description": "The type of storage capacity - StorageCapacity_t"
- },
- "TotalSize": {
- "type": "integer",
- "description": "The total amount of storage (GB)"
- },
- "FreeSize": {
- "type": "integer",
- "description": "The amount of available storage (GB)"
- },
- "UsedSize": {
- "type": "integer",
- "description": "The amount of used storage (GB)"
- },
- "ReservedSize": {
- "type": "integer",
- "description": "The amount storage which is not occupied, but has been reserved for use (GB)"
- },
- "StorageShareID": {
- "type": "string",
- "description": "The ID of the StorageShare related to this capacity"
- }
- },
- "required": ["Type","StorageShareID"]
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/ToComputingService.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/ToComputingService.json b/modules/gfac/gfac-monitor/src/main/resources/schema/ToComputingService.json
deleted file mode 100644
index 6d81b80..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/ToComputingService.json
+++ /dev/null
@@ -1,32 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ToComputingService.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
- "properties": {
- "NetworkInfo": {
- "type": "string",
- "description": "Type of network connections between the Storage and Computing services (NetworkInfo_t)"
- },
- "Bandwidth": {
- "type": "integer",
- "description": "The normal bandwidth available between the Storage and Computing services (Mb/s)"
- },
- "StorageAccessProtocolID": {
- "type": "array",
- "description": "IDs of the protocols that can be used to access the StorageService",
- "items": {
- "type": "string"
- }
- },
- "ComputingServiceID": {
- "type": "string",
- "description": "The ID of the ComputingService"
- },
- "StorageServiceID": {
- "type": "string",
- "description": "The ID of the StorageService"
- }
- },
- "required": ["ComputingServiceID","StorageServiceID"]
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/ToStorageService.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/ToStorageService.json b/modules/gfac/gfac-monitor/src/main/resources/schema/ToStorageService.json
deleted file mode 100644
index 644f3d1..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/ToStorageService.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ToStorageService.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
- "properties": {
- "LocalPath": {
- "type": "string",
- "description": "The path within the ComputingService that is used to access the StorageService"
- },
- "RemotePath": {
- "type": "string",
- "description": "The path in the StorageService which is associated with the LocalPath"
- },
- "ComputingServiceID": {
- "type": "string",
- "description": "The ID of the ComputingService"
- },
- "StorageServiceID": {
- "type": "string",
- "description": "The ID of the StorageService"
- }
- },
- "required": ["LocalPath","RemotePath","ComputingServiceID","StorageServiceID"]
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/UserDomain.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/UserDomain.json b/modules/gfac/gfac-monitor/src/main/resources/schema/UserDomain.json
deleted file mode 100644
index 7acda31..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/UserDomain.json
+++ /dev/null
@@ -1,58 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/UserDomain.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Domain.json"}],
- "properties": {
- "Level": {
- "type": "integer",
- "description": "the number of hops to reach the root of the hierarchy of UserDomains"
- },
- "UserManagerID": {
- "type": "array",
- "description": "ID for the Endpoint of a Service managing users in this UserDomain",
- "items": {
- "type": "string"
- }
- },
- "Member": {
- "type": "array",
- "description": "Identifiers for users in this UserDomain",
- "items": {
- "type": "string"
- }
- },
- "PolicyID": {
- "type": "array",
- "description": "IDs for Policies associated with this UserDomain",
- "items": {
- "type": "string"
- }
- },
- "ChildDomainID": {
- "type": "array",
- "description": "IDs of UserDomains aggregated by this UserDomain",
- "items": {
- "type": "string"
- }
- },
- "ParentDomainID": {
- "type": "string",
- "description": "The ID of the UserDomain that this UserDomain participates in"
- },
- "AccessPolicyID": {
- "type": "array",
- "description": "IDs of AccessPolicies associated with this UserDomain",
- "items": {
- "type": "string"
- }
- },
- "MappingPolicyID": {
- "type": "array",
- "description": "IDs of MappingPolicies associated with this UserDomain",
- "items": {
- "type": "string"
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/service.properties
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/service.properties b/modules/gfac/gfac-monitor/src/main/resources/service.properties
deleted file mode 100644
index 391bfea..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/service.properties
+++ /dev/null
@@ -1,58 +0,0 @@
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-
-
-#
-# Class which implemented Scheduler interface. It will be used to determine a Provider
-#
-scheduler.class= org.apache.airavata.core.gfac.scheduler.impl.SchedulerImpl
-
-#
-# Data Service Plugins classes
-#
-datachain.classes= org.apache.airavata.core.gfac.extension.data.RegistryDataService
-
-#
-# Pre execution Plugins classes. For example, GridFTP Input Staging
-#
-prechain.classes= org.apache.airavata.core.gfac.extension.pre.GridFtpInputStaging
-prechain.classes= org.apache.airavata.core.gfac.extension.pre.HttpInputStaging
-
-#
-# Post execution Plugins classes. For example, GridFTP Output Staging
-#
-postchain.classes= org.apache.airavata.core.gfac.extension.post.GridFtpOutputStaging
-postchain.classes= org.apache.airavata.core.gfac.extension.post.OutputRegister
-
-#
-# SSH private key location. It will be used by SSHProvider
-#
-# ssh.key=/home/user/.ssh/id_rsa
-# ssh.keypass=
-# ssh.username=usernameAtHost
-
-#
-# MyProxy credential. It will be used by GridFTP Plugins and GramProvider.
-#
-# myproxy.server=myproxy.teragrid.org
-# myproxy.user=username
-# myproxy.pass=password
-# myproxy.life=3600
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/test/java/org/apache/airavata/job/AMQPMonitorTest.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/test/java/org/apache/airavata/job/AMQPMonitorTest.java b/modules/gfac/gfac-monitor/src/test/java/org/apache/airavata/job/AMQPMonitorTest.java
deleted file mode 100644
index daa2e91..0000000
--- a/modules/gfac/gfac-monitor/src/test/java/org/apache/airavata/job/AMQPMonitorTest.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.job;
-
-import com.google.common.eventbus.EventBus;
-import com.google.common.eventbus.Subscribe;
-import org.airavata.appcatalog.cpi.AppCatalog;
-import org.apache.aiaravata.application.catalog.data.impl.AppCatalogFactory;
-import org.apache.airavata.common.utils.MonitorPublisher;
-import org.apache.airavata.gfac.core.monitor.MonitorID;
-import org.apache.airavata.gfac.monitor.impl.push.amqp.AMQPMonitor;
-import org.apache.airavata.gsi.ssh.api.Cluster;
-import org.apache.airavata.gsi.ssh.api.SSHApiException;
-import org.apache.airavata.gsi.ssh.api.ServerInfo;
-import org.apache.airavata.gsi.ssh.api.authentication.GSIAuthenticationInfo;
-import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
-import org.apache.airavata.gsi.ssh.impl.PBSCluster;
-import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo;
-import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
-import org.apache.airavata.model.appcatalog.computeresource.DataMovementInterface;
-import org.apache.airavata.model.appcatalog.computeresource.DataMovementProtocol;
-import org.apache.airavata.model.appcatalog.computeresource.JobManagerCommand;
-import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionInterface;
-import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
-import org.apache.airavata.model.appcatalog.computeresource.ResourceJobManager;
-import org.apache.airavata.model.appcatalog.computeresource.ResourceJobManagerType;
-import org.apache.airavata.model.appcatalog.computeresource.SSHJobSubmission;
-import org.apache.airavata.model.appcatalog.computeresource.SecurityProtocol;
-import org.apache.airavata.model.messaging.event.JobStatusChangeEvent;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-public class AMQPMonitorTest {
-
- private String myProxyUserName;
- private String myProxyPassword;
- private String certificateLocation;
- private String pbsFilePath;
- private String workingDirectory;
- private MonitorPublisher monitorPublisher;
- private BlockingQueue<MonitorID> finishQueue;
- private BlockingQueue<MonitorID> pushQueue;
- private Thread pushThread;
- private String proxyFilePath;
- private ComputeResourceDescription computeResourceDescription;
- private final static Logger logger = LoggerFactory.getLogger(AMQPMonitorTest.class);
-
- @Before
- public void setUp() throws Exception {
- System.setProperty("myproxy.username", "ogce");
- System.setProperty("myproxy.password", "");
- System.setProperty("basedir", "/Users/lahirugunathilake/work/airavata/sandbox/gsissh");
- System.setProperty("gsi.working.directory", "/home1/01437/ogce");
- System.setProperty("trusted.cert.location", "/Users/lahirugunathilake/Downloads/certificates");
- System.setProperty("proxy.file.path", "/Users/lahirugunathilake/Downloads/x509up_u503876");
- myProxyUserName = System.getProperty("myproxy.username");
- myProxyPassword = System.getProperty("myproxy.password");
- workingDirectory = System.getProperty("gsi.working.directory");
- certificateLocation = System.getProperty("trusted.cert.location");
- proxyFilePath = System.getProperty("proxy.file.path");
- System.setProperty("connection.name", "xsede");
- if (myProxyUserName == null || myProxyPassword == null || workingDirectory == null) {
- System.out.println(">>>>>> Please run tests with my proxy user name and password. " +
- "E.g :- mvn clean install -Dmyproxy.user=xxx -Dmyproxy.password=xxx -Dgsi.working.directory=/path<<<<<<<");
- throw new Exception("Need my proxy user name password to run tests.");
- }
-
- monitorPublisher = new MonitorPublisher(new EventBus());
- pushQueue = new LinkedBlockingQueue<MonitorID>();
- finishQueue = new LinkedBlockingQueue<MonitorID>();
-
-
- final AMQPMonitor amqpMonitor = new
- AMQPMonitor(monitorPublisher,
- pushQueue, finishQueue,proxyFilePath,"xsede",
- Arrays.asList("info1.dyn.teragrid.org,info2.dyn.teragrid.org".split(",")));
- try {
- (new Thread(){
- public void run(){
- amqpMonitor.run();
- }
- }).start();
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- }
- computeResourceDescription = new ComputeResourceDescription("TestComputerResoruceId", "TestHostName");
- computeResourceDescription.setHostName("stampede-host");
- computeResourceDescription.addToIpAddresses("login1.stampede.tacc.utexas.edu");
- ResourceJobManager resourceJobManager = new ResourceJobManager("1234", ResourceJobManagerType.SLURM);
- Map<JobManagerCommand, String> commandMap = new HashMap<JobManagerCommand, String>();
- commandMap.put(JobManagerCommand.SUBMISSION, "test");
- resourceJobManager.setJobManagerCommands(commandMap);
- resourceJobManager.setJobManagerBinPath("/usr/bin/");
- resourceJobManager.setPushMonitoringEndpoint("push"); // TODO - add monitor mode
- SSHJobSubmission sshJobSubmission = new SSHJobSubmission("TestSSHJobSubmissionInterfaceId", SecurityProtocol.GSI,
- resourceJobManager);
-
- AppCatalog appCatalog = AppCatalogFactory.getAppCatalog();
- String jobSubmissionID = appCatalog.getComputeResource().addSSHJobSubmission(sshJobSubmission);
-
- JobSubmissionInterface jobSubmissionInterface = new JobSubmissionInterface(jobSubmissionID, JobSubmissionProtocol.SSH, 1);
-
- computeResourceDescription.addToJobSubmissionInterfaces(jobSubmissionInterface);
- computeResourceDescription.addToDataMovementInterfaces(new DataMovementInterface("4532", DataMovementProtocol.SCP, 1));
-
- }
-
- @Test
- public void testAMQPMonitor() throws SSHApiException {
- /* now have to submit a job to some machine and add that job to the queue */
- //Create authentication
- GSIAuthenticationInfo authenticationInfo
- = new MyProxyAuthenticationInfo(myProxyUserName, myProxyPassword, "myproxy.teragrid.org",
- 7512, 17280000, certificateLocation);
-
- // Server info
- ServerInfo serverInfo = new ServerInfo("ogce", "login1.stampede.tacc.utexas.edu",2222);
-
-
- Cluster pbsCluster = new
- PBSCluster(serverInfo, authenticationInfo, org.apache.airavata.gsi.ssh.util.CommonUtils.getPBSJobManager("/usr/bin/"));
-
-
- // Execute command
- System.out.println("Target PBS file path: " + workingDirectory);
- // constructing the job object
- String jobName = "GSI_SSH_SLEEP_JOB";
- JobDescriptor jobDescriptor = new JobDescriptor();
- jobDescriptor.setWorkingDirectory(workingDirectory);
- jobDescriptor.setShellName("/bin/bash");
- jobDescriptor.setJobName(jobName);
- jobDescriptor.setExecutablePath("/bin/echo");
- jobDescriptor.setAllEnvExport(true);
- jobDescriptor.setMailOptions("n");
- jobDescriptor.setStandardOutFile(workingDirectory + File.separator + "application.out");
- jobDescriptor.setStandardErrorFile(workingDirectory + File.separator + "application.err");
- jobDescriptor.setNodes(1);
- jobDescriptor.setProcessesPerNode(1);
- jobDescriptor.setQueueName("normal");
- jobDescriptor.setMaxWallTime("60");
- jobDescriptor.setAcountString("TG-STA110014S");
- List<String> inputs = new ArrayList<String>();
- jobDescriptor.setOwner("ogce");
- inputs.add("Hello World");
- jobDescriptor.setInputValues(inputs);
- //finished construction of job object
- System.out.println(jobDescriptor.toXML());
- String jobID = pbsCluster.submitBatchJob(jobDescriptor);
- System.out.println(jobID);
- try {
- pushQueue.add(new MonitorID(computeResourceDescription, jobID,null,null,null, "ogce", jobName));
- } catch (Exception e) {
- e.printStackTrace();
- }
- try {
- pushThread.join();
- } catch (InterruptedException e) {
- e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
- }
- class InnerClassAMQP{
- @Subscribe
- private void getStatus(JobStatusChangeEvent status){
- Assert.assertNotNull(status);
- pushThread.interrupt();
- }
- }
- monitorPublisher.registerListener(new InnerClassAMQP());
-// try {
-// pushThread.join(5000);
-// Iterator<MonitorID> iterator = pushQueue.iterator();
-// MonitorID next = iterator.next();
-// org.junit.Assert.assertNotNull(next.getStatus());
-// } catch (Exception e) {
-// e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
-// }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/test/java/org/apache/airavata/job/QstatMonitorTestWithMyProxyAuth.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/test/java/org/apache/airavata/job/QstatMonitorTestWithMyProxyAuth.java b/modules/gfac/gfac-monitor/src/test/java/org/apache/airavata/job/QstatMonitorTestWithMyProxyAuth.java
deleted file mode 100644
index 610934e..0000000
--- a/modules/gfac/gfac-monitor/src/test/java/org/apache/airavata/job/QstatMonitorTestWithMyProxyAuth.java
+++ /dev/null
@@ -1,172 +0,0 @@
-///*
-// *
-// * Licensed to the Apache Software Foundation (ASF) under one
-// * or more contributor license agreements. See the NOTICE file
-// * distributed with this work for additional information
-// * regarding copyright ownership. The ASF licenses this file
-// * to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance
-// * with the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing,
-// * software distributed under the License is distributed on an
-// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// * KIND, either express or implied. See the License for the
-// * specific language governing permissions and limitations
-// * under the License.
-// *
-//*/
-//package org.apache.airavata.job;
-//
-//import java.io.File;
-//import java.util.ArrayList;
-//import java.util.List;
-//import java.util.concurrent.BlockingQueue;
-//import java.util.concurrent.LinkedBlockingQueue;
-//
-//import org.apache.airavata.common.utils.MonitorPublisher;
-//import org.apache.airavata.commons.gfac.type.HostDescription;
-//import org.apache.airavata.gfac.core.monitor.MonitorID;
-//import org.apache.airavata.gfac.monitor.HPCMonitorID;
-//import org.apache.airavata.gfac.monitor.UserMonitorData;
-//import org.apache.airavata.gfac.monitor.impl.pull.qstat.HPCPullMonitor;
-//import org.apache.airavata.gsi.ssh.api.Cluster;
-//import org.apache.airavata.gsi.ssh.api.SSHApiException;
-//import org.apache.airavata.gsi.ssh.api.ServerInfo;
-//import org.apache.airavata.gsi.ssh.api.authentication.GSIAuthenticationInfo;
-//import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
-//import org.apache.airavata.gsi.ssh.impl.PBSCluster;
-//import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo;
-//import org.apache.airavata.gsi.ssh.util.CommonUtils;
-//import org.apache.airavata.model.messaging.event.JobStatusChangeEvent;
-//import org.apache.airavata.schemas.gfac.GsisshHostType;
-//import org.junit.Assert;
-//import org.testng.annotations.Test;
-//
-//import com.google.common.eventbus.EventBus;
-//import com.google.common.eventbus.Subscribe;
-//
-//public class QstatMonitorTestWithMyProxyAuth {
-// private String myProxyUserName;
-// private String myProxyPassword;
-// private String certificateLocation;
-// private String pbsFilePath;
-// private String workingDirectory;
-// private HostDescription hostDescription;
-// private MonitorPublisher monitorPublisher;
-// private BlockingQueue<UserMonitorData> pullQueue;
-// private Thread monitorThread;
-//
-// @org.testng.annotations.BeforeClass
-// public void setUp() throws Exception {
-//// System.setProperty("myproxy.username", "ogce");
-//// System.setProperty("myproxy.password", "");
-//// System.setProperty("basedir", "/Users/lahirugunathilake/work/airavata/sandbox/gsissh");
-//// System.setProperty("gsi.working.directory", "/home/ogce");
-//// System.setProperty("trusted.cert.location", "/Users/lahirugunathilake/Downloads/certificates");
-// myProxyUserName = System.getProperty("myproxy.username");
-// myProxyPassword = System.getProperty("myproxy.password");
-// workingDirectory = System.getProperty("gsi.working.directory");
-// certificateLocation = System.getProperty("trusted.cert.location");
-// if (myProxyUserName == null || myProxyPassword == null || workingDirectory == null) {
-// System.out.println(">>>>>> Please run tests with my proxy user name and password. " +
-// "E.g :- mvn clean install -Dmyproxy.username=xxx -Dmyproxy.password=xxx -Dgsi.working.directory=/path<<<<<<<");
-// throw new Exception("Need my proxy user name password to run tests.");
-// }
-//
-// monitorPublisher = new MonitorPublisher(new EventBus());
-// class InnerClassQstat {
-//
-// @Subscribe
-// private void getStatus(JobStatusChangeEvent status) {
-// Assert.assertNotNull(status);
-// System.out.println(status.getState().toString());
-// monitorThread.interrupt();
-// }
-// }
-// monitorPublisher.registerListener(this);
-// pullQueue = new LinkedBlockingQueue<UserMonitorData>();
-// final HPCPullMonitor qstatMonitor = new
-// HPCPullMonitor(pullQueue, monitorPublisher);
-// try {
-// (new Thread(){
-// public void run(){
-// qstatMonitor.run();
-// }
-// }).start();
-// } catch (Exception e) {
-// e.printStackTrace();
-// }
-//
-// hostDescription = new HostDescription(GsisshHostType.type);
-// hostDescription.getType().setHostAddress("trestles.sdsc.edu");
-// hostDescription.getType().setHostName("gsissh-gordon");
-// ((GsisshHostType) hostDescription.getType()).setPort(22);
-// ((GsisshHostType)hostDescription.getType()).setInstalledPath("/opt/torque/bin/");
-// }
-//
-// @Test
-// public void testQstatMonitor() throws SSHApiException {
-// /* now have to submit a job to some machine and add that job to the queue */
-// //Create authentication
-// GSIAuthenticationInfo authenticationInfo
-// = new MyProxyAuthenticationInfo(myProxyUserName, myProxyPassword, "myproxy.teragrid.org",
-// 7512, 17280000, certificateLocation);
-//
-// // Server info
-// ServerInfo serverInfo = new ServerInfo("ogce", hostDescription.getType().getHostAddress());
-//
-//
-// Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, CommonUtils.getPBSJobManager("/opt/torque/bin/"));
-//
-//
-// // Execute command
-// System.out.println("Target PBS file path: " + workingDirectory);
-// // constructing the job object
-// JobDescriptor jobDescriptor = new JobDescriptor();
-// jobDescriptor.setWorkingDirectory(workingDirectory);
-// jobDescriptor.setShellName("/bin/bash");
-// jobDescriptor.setJobName("GSI_SSH_SLEEP_JOB");
-// jobDescriptor.setExecutablePath("/bin/echo");
-// jobDescriptor.setAllEnvExport(true);
-// jobDescriptor.setMailOptions("n");
-// jobDescriptor.setStandardOutFile(workingDirectory + File.separator + "application.out");
-// jobDescriptor.setStandardErrorFile(workingDirectory + File.separator + "application.err");
-// jobDescriptor.setNodes(1);
-// jobDescriptor.setProcessesPerNode(1);
-// jobDescriptor.setQueueName("normal");
-// jobDescriptor.setMaxWallTime("60");
-// jobDescriptor.setAcountString("sds128");
-// List<String> inputs = new ArrayList<String>();
-// jobDescriptor.setOwner("ogce");
-// inputs.add("Hello World");
-// jobDescriptor.setInputValues(inputs);
-// //finished construction of job object
-// System.out.println(jobDescriptor.toXML());
-// for (int i = 0; i < 1; i++) {
-// String jobID = pbsCluster.submitBatchJob(jobDescriptor);
-// System.out.println("Job submitted successfully, Job ID: " + jobID);
-// MonitorID monitorID = new HPCMonitorID(hostDescription, jobID,null,null,null, "ogce","");
-// ((HPCMonitorID)monitorID).setAuthenticationInfo(authenticationInfo);
-// try {
-// org.apache.airavata.gfac.monitor.util.CommonUtils.addMonitortoQueue(pullQueue, monitorID, jobExecutionContext);
-// } catch (Exception e) {
-// e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
-// }
-// }
-// try {
-//
-// monitorThread.join();
-// } catch (Exception e) {
-// e.printStackTrace();
-// }
-// }
-//
-// @Subscribe
-// public void testCaseShutDown(JobStatusChangeEvent status) {
-// Assert.assertNotNull(status.getState());
-// monitorThread.stop();
-// }
-//}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/test/resources/PBSTemplate.xslt
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/test/resources/PBSTemplate.xslt b/modules/gfac/gfac-monitor/src/test/resources/PBSTemplate.xslt
deleted file mode 100644
index e749e9c..0000000
--- a/modules/gfac/gfac-monitor/src/test/resources/PBSTemplate.xslt
+++ /dev/null
@@ -1,73 +0,0 @@
-<!--Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to you under
- the Apache License, Version 2.0 (theÏ "License"); you may not use this file except in compliance with the License. You may
- obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to
- in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
- ANY ~ KIND, either express or implied. See the License for the specific language governing permissions and limitations under
- the License. -->
-<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:ns="http://airavata.apache.org/gsi/ssh/2012/12">
-<xsl:output method="text" />
-<xsl:template match="/ns:JobDescriptor">
-#! /bin/sh
-# PBS batch job script built by Globus job manager
-# <xsl:choose>
- <xsl:when test="ns:shellName">
-##PBS -S <xsl:value-of select="ns:shellName"/>
- </xsl:when></xsl:choose>
- <xsl:choose>
- <xsl:when test="ns:queueName">
-#PBS -q <xsl:value-of select="ns:queueName"/>
- </xsl:when>
- </xsl:choose>
- <xsl:choose>
- <xsl:when test="ns:mailOptions">
-#PBS -m <xsl:value-of select="ns:mailOptions"/>
- </xsl:when>
- </xsl:choose>
- <xsl:choose>
-<xsl:when test="ns:acountString">
-#PBS -A <xsl:value-of select="ns:acountString"/>
- </xsl:when>
- </xsl:choose>
- <xsl:choose>
- <xsl:when test="ns:maxWallTime">
-#PBS -l walltime=<xsl:value-of select="ns:maxWallTime"/>
- </xsl:when>
- </xsl:choose>
- <xsl:choose>
- <xsl:when test="ns:standardOutFile">
-#PBS -o <xsl:value-of select="ns:standardOutFile"/>
- </xsl:when>
- </xsl:choose>
- <xsl:choose>
- <xsl:when test="ns:standardOutFile">
-#PBS -e <xsl:value-of select="ns:standardErrorFile"/>
- </xsl:when>
- </xsl:choose>
- <xsl:choose>
- <xsl:when test="(ns:nodes) and (ns:processesPerNode)">
-#PBS -l nodes=<xsl:value-of select="ns:nodes"/>:ppn=<xsl:value-of select="ns:processesPerNode"/>
-<xsl:text>
</xsl:text>
- </xsl:when>
- </xsl:choose>
-<xsl:for-each select="ns:exports/ns:name">
-<xsl:value-of select="."/>=<xsl:value-of select="./@value"/><xsl:text>
</xsl:text>
-export<xsl:text> </xsl:text><xsl:value-of select="."/>
-<xsl:text>
</xsl:text>
-</xsl:for-each>
-<xsl:for-each select="ns:preJobCommands/ns:command">
- <xsl:value-of select="."/><xsl:text> </xsl:text>
- </xsl:for-each>
-cd <xsl:text> </xsl:text><xsl:value-of select="ns:workingDirectory"/><xsl:text>
</xsl:text>
- <xsl:choose><xsl:when test="ns:jobSubmitterCommand">
-<xsl:value-of select="ns:jobSubmitterCommand"/><xsl:text> </xsl:text></xsl:when></xsl:choose><xsl:value-of select="ns:executablePath"/><xsl:text> </xsl:text>
-<xsl:for-each select="ns:inputs/ns:input">
- <xsl:value-of select="."/><xsl:text> </xsl:text>
- </xsl:for-each>
-<xsl:for-each select="ns:postJobCommands/ns:command">
- <xsl:value-of select="."/><xsl:text> </xsl:text>
-</xsl:for-each>
-
-</xsl:template>
-
-</xsl:stylesheet>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/test/resources/echo.bat
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/test/resources/echo.bat b/modules/gfac/gfac-monitor/src/test/resources/echo.bat
deleted file mode 100644
index c6b849b..0000000
--- a/modules/gfac/gfac-monitor/src/test/resources/echo.bat
+++ /dev/null
@@ -1,22 +0,0 @@
-::
-::
-:: Licensed to the Apache Software Foundation (ASF) under one
-:: or more contributor license agreements. See the NOTICE file
-:: distributed with this work for additional information
-:: regarding copyright ownership. The ASF licenses this file
-:: to you under the Apache License, Version 2.0 (the
-:: "License"); you may not use this file except in compliance
-:: with the License. You may obtain a copy of the License at
-::
-:: http://www.apache.org/licenses/LICENSE-2.0
-::
-:: Unless required by applicable law or agreed to in writing,
-:: software distributed under the License is distributed on an
-:: "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-:: KIND, either express or implied. See the License for the
-:: specific language governing permissions and limitations
-:: under the License.
-::
-::
-@echo off
-echo %1^=%2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/test/resources/logging.properties
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/test/resources/logging.properties b/modules/gfac/gfac-monitor/src/test/resources/logging.properties
deleted file mode 100644
index 0584d38..0000000
--- a/modules/gfac/gfac-monitor/src/test/resources/logging.properties
+++ /dev/null
@@ -1,42 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-#default/fallback log4j configuration
-#
-
-# Set root logger level to WARN and its only appender to A1.
-log4j.rootLogger=INFO, A1, A2
-
-# A1 is set to be a rolling file appender with default params
-log4j.appender.A1=org.apache.log4j.RollingFileAppender
-log4j.appender.A1.File=target/seclogs.txt
-
-# A1 uses PatternLayout.
-log4j.appender.A1.layout=org.apache.log4j.PatternLayout
-log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %c %x - %m%n
-
-# A2 is a console appender
-log4j.appender.A2=org.apache.log4j.ConsoleAppender
-
-# A2 uses PatternLayout.
-log4j.appender.A2.layout=org.apache.log4j.PatternLayout
-log4j.appender.A2.layout.ConversionPattern=%d [%t] %-5p %c{1} %x - %m%n
-
-log4j.logger.unicore.security=INFO
-
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-ssh/pom.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/pom.xml b/modules/gfac/gfac-ssh/pom.xml
index a7c97f1..b0f9d68 100644
--- a/modules/gfac/gfac-ssh/pom.xml
+++ b/modules/gfac/gfac-ssh/pom.xml
@@ -23,6 +23,13 @@
<url>http://airavata.apache.org/</url>
<dependencies>
+
+ <!--email monitoring-->
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-gfac-email-monitor</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<!-- Logging -->
<dependency>
<groupId>org.slf4j</groupId>
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java
index 32be550..b201c79 100644
--- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java
+++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java
@@ -35,7 +35,7 @@ import org.apache.airavata.gfac.core.notification.events.StartExecutionEvent;
import org.apache.airavata.gfac.core.provider.AbstractProvider;
import org.apache.airavata.gfac.core.provider.GFacProviderException;
import org.apache.airavata.gfac.core.utils.GFacUtils;
-import org.apache.airavata.gfac.core.monitor.mail.EmailBasedMonitor;
+import org.apache.airavata.gfac.monitor.email.EmailBasedMonitor;
import org.apache.airavata.gfac.ssh.security.SSHSecurityContext;
import org.apache.airavata.gfac.ssh.util.GFACSSHUtils;
import org.apache.airavata.gsi.ssh.api.Cluster;
[7/7] airavata git commit: Add new gfac email monitor module to gfac
monitor module and Intorduced gfac-hpc-monitor module to keep all hoc-monitor
code
Posted by sh...@apache.org.
Add new gfac email monitor module to gfac monitor module and Intorduced gfac-hpc-monitor module to keep all hoc-monitor code
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/6ffc3ee7
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/6ffc3ee7
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/6ffc3ee7
Branch: refs/heads/emailMonitoring
Commit: 6ffc3ee70114f704e6ec7adcdd05c471d98517f0
Parents: 95fae20
Author: shamrath <sh...@gmail.com>
Authored: Wed Apr 8 11:24:32 2015 -0400
Committer: shamrath <sh...@gmail.com>
Committed: Wed Apr 8 11:24:32 2015 -0400
----------------------------------------------------------------------
.../core/monitor/mail/EmailBasedMonitor.java | 224 ---------
.../gfac/core/monitor/mail/JobStatusResult.java | 44 --
.../core/monitor/mail/parser/EmailParser.java | 35 --
.../mail/parser/LonestarEmailParser.java | 74 ---
.../monitor/mail/parser/PBSEmailParser.java | 139 ------
.../monitor/mail/parser/SLURMEmailParser.java | 101 ----
.../gfac-monitor/gfac-email-monitor/pom.xml | 37 ++
.../gfac/monitor/email/EmailBasedMonitor.java | 229 +++++++++
.../gfac/monitor/email/JobStatusResult.java | 44 ++
.../gfac/monitor/email/parser/EmailParser.java | 35 ++
.../email/parser/LonestarEmailParser.java | 74 +++
.../monitor/email/parser/PBSEmailParser.java | 139 ++++++
.../monitor/email/parser/SLURMEmailParser.java | 101 ++++
.../gfac/gfac-monitor/gfac-hpc-monitor/pom.xml | 165 +++++++
.../airavata/gfac/monitor/HPCMonitorID.java | 107 +++++
.../airavata/gfac/monitor/HostMonitorData.java | 88 ++++
.../airavata/gfac/monitor/UserMonitorData.java | 76 +++
.../command/ExperimentCancelRequest.java | 38 ++
.../gfac/monitor/command/TaskCancelRequest.java | 52 ++
.../monitor/core/AiravataAbstractMonitor.java | 46 ++
.../gfac/monitor/core/MessageParser.java | 43 ++
.../airavata/gfac/monitor/core/Monitor.java | 30 ++
.../airavata/gfac/monitor/core/PullMonitor.java | 64 +++
.../airavata/gfac/monitor/core/PushMonitor.java | 60 +++
.../exception/AiravataMonitorException.java | 37 ++
.../handlers/GridPullMonitorHandler.java | 142 ++++++
.../handlers/GridPushMonitorHandler.java | 103 ++++
.../monitor/impl/pull/qstat/HPCPullMonitor.java | 469 +++++++++++++++++++
.../impl/pull/qstat/ResourceConnection.java | 154 ++++++
.../monitor/impl/push/amqp/AMQPMonitor.java | 280 +++++++++++
.../monitor/impl/push/amqp/BasicConsumer.java | 87 ++++
.../impl/push/amqp/JSONMessageParser.java | 78 +++
.../impl/push/amqp/SimpleJobFinishConsumer.java | 86 ++++
.../impl/push/amqp/UnRegisterWorker.java | 67 +++
.../gfac/monitor/util/AMQPConnectionUtil.java | 80 ++++
.../airavata/gfac/monitor/util/CommonUtils.java | 298 ++++++++++++
.../airavata/gfac/monitor/util/X509Helper.java | 164 +++++++
.../src/main/resources/errors.properties | 197 ++++++++
.../src/main/resources/schema/AccessPolicy.json | 13 +
.../src/main/resources/schema/Activity.json | 31 ++
.../src/main/resources/schema/AdminDomain.json | 51 ++
.../schema/ApplicationEnvironment.json | 86 ++++
.../resources/schema/ApplicationHandle.json | 21 +
.../src/main/resources/schema/Benchmark.json | 21 +
.../resources/schema/ComputingActivity.json | 165 +++++++
.../resources/schema/ComputingEndpoint.json | 44 ++
.../main/resources/schema/ComputingManager.json | 117 +++++
.../main/resources/schema/ComputingService.json | 32 ++
.../main/resources/schema/ComputingShare.json | 182 +++++++
.../src/main/resources/schema/Contact.json | 32 ++
.../src/main/resources/schema/DataStore.json | 30 ++
.../src/main/resources/schema/Domain.json | 30 ++
.../src/main/resources/schema/Endpoint.json | 147 ++++++
.../src/main/resources/schema/Entity.json | 35 ++
.../resources/schema/ExecutionEnvironment.json | 115 +++++
.../src/main/resources/schema/Glue2.json | 246 ++++++++++
.../src/main/resources/schema/Location.json | 47 ++
.../src/main/resources/schema/Manager.json | 28 ++
.../main/resources/schema/MappingPolicy.json | 13 +
.../src/main/resources/schema/Policy.json | 27 ++
.../src/main/resources/schema/Resource.json | 27 ++
.../src/main/resources/schema/Service.json | 75 +++
.../src/main/resources/schema/Share.json | 45 ++
.../resources/schema/StorageAccessProtocol.json | 32 ++
.../main/resources/schema/StorageEndpoint.json | 8 +
.../main/resources/schema/StorageManager.json | 8 +
.../main/resources/schema/StorageService.json | 22 +
.../schema/StorageServiceCapacity.json | 33 ++
.../src/main/resources/schema/StorageShare.json | 65 +++
.../resources/schema/StorageShareCapacity.json | 33 ++
.../resources/schema/ToComputingService.json | 32 ++
.../main/resources/schema/ToStorageService.json | 25 +
.../src/main/resources/schema/UserDomain.json | 58 +++
.../src/main/resources/service.properties | 58 +++
.../apache/airavata/job/AMQPMonitorTest.java | 207 ++++++++
.../job/QstatMonitorTestWithMyProxyAuth.java | 172 +++++++
.../src/test/resources/PBSTemplate.xslt | 73 +++
.../src/test/resources/echo.bat | 22 +
.../src/test/resources/logging.properties | 42 ++
modules/gfac/gfac-monitor/pom.xml | 158 +------
.../airavata/gfac/monitor/HPCMonitorID.java | 107 -----
.../airavata/gfac/monitor/HostMonitorData.java | 88 ----
.../airavata/gfac/monitor/UserMonitorData.java | 76 ---
.../command/ExperimentCancelRequest.java | 38 --
.../gfac/monitor/command/TaskCancelRequest.java | 52 --
.../monitor/core/AiravataAbstractMonitor.java | 46 --
.../gfac/monitor/core/MessageParser.java | 43 --
.../airavata/gfac/monitor/core/Monitor.java | 30 --
.../airavata/gfac/monitor/core/PullMonitor.java | 64 ---
.../airavata/gfac/monitor/core/PushMonitor.java | 60 ---
.../exception/AiravataMonitorException.java | 37 --
.../handlers/GridPullMonitorHandler.java | 142 ------
.../handlers/GridPushMonitorHandler.java | 103 ----
.../monitor/impl/pull/qstat/HPCPullMonitor.java | 469 -------------------
.../impl/pull/qstat/ResourceConnection.java | 154 ------
.../monitor/impl/push/amqp/AMQPMonitor.java | 280 -----------
.../monitor/impl/push/amqp/BasicConsumer.java | 87 ----
.../impl/push/amqp/JSONMessageParser.java | 78 ---
.../impl/push/amqp/SimpleJobFinishConsumer.java | 86 ----
.../impl/push/amqp/UnRegisterWorker.java | 67 ---
.../gfac/monitor/util/AMQPConnectionUtil.java | 80 ----
.../airavata/gfac/monitor/util/CommonUtils.java | 298 ------------
.../airavata/gfac/monitor/util/X509Helper.java | 164 -------
.../src/main/resources/errors.properties | 197 --------
.../src/main/resources/schema/AccessPolicy.json | 13 -
.../src/main/resources/schema/Activity.json | 31 --
.../src/main/resources/schema/AdminDomain.json | 51 --
.../schema/ApplicationEnvironment.json | 86 ----
.../resources/schema/ApplicationHandle.json | 21 -
.../src/main/resources/schema/Benchmark.json | 21 -
.../resources/schema/ComputingActivity.json | 165 -------
.../resources/schema/ComputingEndpoint.json | 44 --
.../main/resources/schema/ComputingManager.json | 117 -----
.../main/resources/schema/ComputingService.json | 32 --
.../main/resources/schema/ComputingShare.json | 182 -------
.../src/main/resources/schema/Contact.json | 32 --
.../src/main/resources/schema/DataStore.json | 30 --
.../src/main/resources/schema/Domain.json | 30 --
.../src/main/resources/schema/Endpoint.json | 147 ------
.../src/main/resources/schema/Entity.json | 35 --
.../resources/schema/ExecutionEnvironment.json | 115 -----
.../src/main/resources/schema/Glue2.json | 246 ----------
.../src/main/resources/schema/Location.json | 47 --
.../src/main/resources/schema/Manager.json | 28 --
.../main/resources/schema/MappingPolicy.json | 13 -
.../src/main/resources/schema/Policy.json | 27 --
.../src/main/resources/schema/Resource.json | 27 --
.../src/main/resources/schema/Service.json | 75 ---
.../src/main/resources/schema/Share.json | 45 --
.../resources/schema/StorageAccessProtocol.json | 32 --
.../main/resources/schema/StorageEndpoint.json | 8 -
.../main/resources/schema/StorageManager.json | 8 -
.../main/resources/schema/StorageService.json | 22 -
.../schema/StorageServiceCapacity.json | 33 --
.../src/main/resources/schema/StorageShare.json | 65 ---
.../resources/schema/StorageShareCapacity.json | 33 --
.../resources/schema/ToComputingService.json | 32 --
.../main/resources/schema/ToStorageService.json | 25 -
.../src/main/resources/schema/UserDomain.json | 58 ---
.../src/main/resources/service.properties | 58 ---
.../apache/airavata/job/AMQPMonitorTest.java | 207 --------
.../job/QstatMonitorTestWithMyProxyAuth.java | 172 -------
.../src/test/resources/PBSTemplate.xslt | 73 ---
.../gfac-monitor/src/test/resources/echo.bat | 22 -
.../src/test/resources/logging.properties | 42 --
modules/gfac/gfac-ssh/pom.xml | 7 +
.../gfac/ssh/provider/impl/SSHProvider.java | 2 +-
147 files changed, 6235 insertions(+), 6165 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/EmailBasedMonitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/EmailBasedMonitor.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/EmailBasedMonitor.java
deleted file mode 100644
index f15e582..0000000
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/EmailBasedMonitor.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.core.monitor.mail;
-
-import org.apache.airavata.common.exception.AiravataException;
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.logger.AiravataLogger;
-import org.apache.airavata.common.logger.AiravataLoggerFactory;
-import org.apache.airavata.common.utils.MonitorPublisher;
-import org.apache.airavata.common.utils.ServerSettings;
-import org.apache.airavata.gfac.core.context.JobExecutionContext;
-import org.apache.airavata.gfac.core.utils.GFacThreadPoolExecutor;
-import org.apache.airavata.gfac.core.utils.OutHandlerWorker;
-import org.apache.airavata.gfac.core.monitor.mail.parser.EmailParser;
-import org.apache.airavata.gfac.core.monitor.mail.parser.PBSEmailParser;
-import org.apache.airavata.gfac.core.monitor.mail.parser.SLURMEmailParser;
-import org.apache.airavata.model.messaging.event.JobIdentifier;
-import org.apache.airavata.model.messaging.event.JobStatusChangeRequestEvent;
-import org.apache.airavata.model.workspace.experiment.JobState;
-import org.apache.airavata.model.workspace.experiment.JobStatus;
-
-import javax.mail.Address;
-import javax.mail.Flags;
-import javax.mail.Folder;
-import javax.mail.Message;
-import javax.mail.MessagingException;
-import javax.mail.Session;
-import javax.mail.Store;
-import javax.mail.search.FlagTerm;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class EmailBasedMonitor implements Runnable{
-
- private static final AiravataLogger log = AiravataLoggerFactory.getLogger(EmailBasedMonitor.class);
-
- private static final String PBS_CONSULT_SDSC_EDU = "pbsconsult@sdsc.edu";
- private static final String SLURM_BATCH_STAMPEDE = "slurm@batch1.stampede.tacc.utexas.edu";
- private static EmailBasedMonitor emailBasedMonitor;
- private final MonitorPublisher monitorPublisher;
-
- private Session session ;
- private Store store;
- private Folder emailFolder;
- private String host, emailAddress, password, folderName, mailStoreProtocol;
- private Properties properties;
-
- private Map<String, JobExecutionContext> jobMonitorMap = new ConcurrentHashMap<String, JobExecutionContext>();
-
- private EmailBasedMonitor(MonitorPublisher monitorPublisher) throws ApplicationSettingsException {
- this.monitorPublisher = monitorPublisher;
- init();
- }
-
- private void init() throws ApplicationSettingsException {
- host = ServerSettings.getEmailBasedMonitorHost();
- emailAddress = ServerSettings.getEmailBasedMonitorAddress();
- password = ServerSettings.getEmailBasedMonitorPassword();
- folderName = ServerSettings.getEmailBasedMonitorFolderName();
- mailStoreProtocol = ServerSettings.getEmailBasedMonitorStoreProtocol();
-
- properties = new Properties();
- properties.put("mail.store.protocol", mailStoreProtocol);
-
- }
-
- public static EmailBasedMonitor getInstant(MonitorPublisher monitorPublisher) throws ApplicationSettingsException {
- if (emailBasedMonitor == null) {
- synchronized (EmailBasedMonitor.class) {
- if (emailBasedMonitor == null) {
- emailBasedMonitor = new EmailBasedMonitor(monitorPublisher);
- Thread thread = new Thread(emailBasedMonitor);
- thread.start();
- }
- }
- }
-
- return emailBasedMonitor;
- }
-
- public void addToJobMonitorMap(JobExecutionContext jobExecutionContext) {
- addToJobMonitorMap(jobExecutionContext.getJobDetails().getJobID(), jobExecutionContext);
- }
-
- public void addToJobMonitorMap(String jobId, JobExecutionContext jobExecutionContext) {
- log.info("Added Job Id : " + jobId + " to email based monitor map");
- jobMonitorMap.put(jobId, jobExecutionContext);
- }
-
- private JobStatusResult parse(Message message) throws MessagingException, AiravataException {
- Address fromAddress = message.getFrom()[0];
- EmailParser emailParser;
- String addressStr = fromAddress.toString();
- switch (addressStr) {
- case PBS_CONSULT_SDSC_EDU:
- emailParser = new PBSEmailParser();
- break;
- case SLURM_BATCH_STAMPEDE:
- emailParser = new SLURMEmailParser();
- break;
- default:
- throw new AiravataException("Un-handle address type for email monitoring --> " + addressStr);
- }
- return emailParser.parseEmail(message);
- }
-
- @Override
- public void run() {
- try {
- session = Session.getDefaultInstance(properties);
- store = session.getStore(mailStoreProtocol);
- store.connect(host, emailAddress, password);
- while (!ServerSettings.isStopAllThreads()) {
- if (!store.isConnected()) {
- store.connect();
- }
- Thread.sleep(2000);
- emailFolder = store.getFolder(folderName);
- emailFolder.open(Folder.READ_WRITE);
- Message[] searchMessages = emailFolder.search(new FlagTerm(new Flags(Flags.Flag.SEEN), false));
- for (Message message : searchMessages) {
- try {
- JobStatusResult jobStatusResult = parse(message);
- process(jobStatusResult);
- } catch (AiravataException e) {
- log.error("Error parsing email message =====================================>", e);
- try {
- writeEnvelopeOnError(message);
- } catch (MessagingException e1) {
- log.error("Error printing envelop of the email");
- }
-
- }
- }
- emailFolder.setFlags(searchMessages, new Flags(Flags.Flag.SEEN), true);
- emailFolder.close(false);
- }
-
- } catch (MessagingException e) {
- log.error("Couldn't connect to the store ", e);
- } catch (InterruptedException e) {
- log.error("Interrupt exception while sleep ", e);
- } finally {
- try {
- store.close();
- } catch (MessagingException e) {
- log.error("Store close operation failed, couldn't close store", e);
- }
- }
- }
-
- private void process(JobStatusResult jobStatusResult) throws AiravataException {
- JobExecutionContext jEC = jobMonitorMap.get(jobStatusResult.getJobId());
- if (jEC == null) {
- throw new AiravataException("JobExecutionContext is not found for job Id " + jobStatusResult.getJobId());
- }
- JobState resultState = jobStatusResult.getState();
- jEC.getJobDetails().setJobStatus(new JobStatus(resultState));
- if (resultState == JobState.COMPLETE) {
- GFacThreadPoolExecutor.getFixedThreadPool().submit(new OutHandlerWorker(jEC, monitorPublisher));
- }else if (resultState == JobState.QUEUED) {
- // TODO - publish queued rabbitmq message
- }else if (resultState == JobState.FAILED) {
- // TODO - handle failed scenario
- jobMonitorMap.remove(jobStatusResult.getJobId());
- log.info("Job failed email received , removed job from job monitoring");
-// monitorPublisher.publish(jEC.getJobDetails().getJobStatus());
- }
- publishJobStatusChange(jEC);
- }
-
- private void publishJobStatusChange(JobExecutionContext jobExecutionContext) {
- JobStatusChangeRequestEvent jobStatus = new JobStatusChangeRequestEvent();
- JobIdentifier jobIdentity = new JobIdentifier(jobExecutionContext.getJobDetails().getJobID(),
- jobExecutionContext.getTaskData().getTaskID(),
- jobExecutionContext.getWorkflowNodeDetails().getNodeInstanceId(),
- jobExecutionContext.getExperimentID(),
- jobExecutionContext.getGatewayID());
- jobStatus.setJobIdentity(jobIdentity);
- jobStatus.setState(jobExecutionContext.getJobDetails().getJobStatus().getJobState());
- // we have this JobStatus class to handle amqp monitoring
- log.debugId(jobStatus.getJobIdentity().getJobId(), "Published job status change request, " +
- "experiment {} , task {}", jobStatus.getJobIdentity().getExperimentId(),
- jobStatus.getJobIdentity().getTaskId());
-
- monitorPublisher.publish(jobStatus);
- }
-
- private void writeEnvelopeOnError(Message m) throws MessagingException {
- Address[] a;
- // FROM
- if ((a = m.getFrom()) != null) {
- for (int j = 0; j < a.length; j++)
- log.error("FROM: " + a[j].toString());
- }
- // TO
- if ((a = m.getRecipients(Message.RecipientType.TO)) != null) {
- for (int j = 0; j < a.length; j++)
- log.error("TO: " + a[j].toString());
- }
- // SUBJECT
- if (m.getSubject() != null)
- log.error("SUBJECT: " + m.getSubject());
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/JobStatusResult.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/JobStatusResult.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/JobStatusResult.java
deleted file mode 100644
index 0c549a6..0000000
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/JobStatusResult.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.core.monitor.mail;
-
-import org.apache.airavata.model.workspace.experiment.JobState;
-
-public class JobStatusResult {
- private JobState state;
- private String jobId;
-
- public JobState getState() {
- return state;
- }
-
- public void setState(JobState state) {
- this.state = state;
- }
-
- public String getJobId() {
- return jobId;
- }
-
- public void setJobId(String jobId) {
- this.jobId = jobId;
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/EmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/EmailParser.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/EmailParser.java
deleted file mode 100644
index ad8e4a3..0000000
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/EmailParser.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.core.monitor.mail.parser;
-
-import org.apache.airavata.common.exception.AiravataException;
-import org.apache.airavata.gfac.core.monitor.mail.JobStatusResult;
-
-import javax.mail.Message;
-import javax.mail.MessagingException;
-
-public interface EmailParser {
- static final String STATUS = "status";
- static final String JOBID = "jobId";
- static final String EXIT_STATUS = "exitStatus";
-
- JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException;
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/LonestarEmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/LonestarEmailParser.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/LonestarEmailParser.java
deleted file mode 100644
index 59651da..0000000
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/LonestarEmailParser.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.core.monitor.mail.parser;
-
-import org.apache.airavata.common.exception.AiravataException;
-import org.apache.airavata.gfac.core.monitor.mail.JobStatusResult;
-import org.apache.airavata.model.workspace.experiment.JobState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.mail.Message;
-import javax.mail.MessagingException;
-import java.io.IOException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class LonestarEmailParser implements EmailParser {
- private static final Logger log = LoggerFactory.getLogger(LonestarEmailParser.class);
- //root@c312-206.ls4.tacc.utexas.edu
- private static final String SIGNAL = "signal";
- private static final String LONESTAR_REGEX = "Job (?<" + JOBID + ">\\d+) \\(.*\\) (?<" + STATUS
- + ">.*)\\s[a-zA-Z =]+(?<" + EXIT_STATUS + ">\\d+)\\sSignal[ ]*=[ ]*(?<" + SIGNAL + ">[a-zA-z]*)";
-
- @Override
- public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException {
- JobStatusResult jobStatusResult = new JobStatusResult();
- try {
- String content = ((String) message.getContent());
- Pattern pattern = Pattern.compile(LONESTAR_REGEX);
- Matcher matcher = pattern.matcher(content);
- if (matcher.find()) {
- jobStatusResult.setJobId(matcher.group(JOBID));
- String status = matcher.group(STATUS);
- jobStatusResult.setState(getJobState(status, content));
- return jobStatusResult;
- } else {
- log.error("No matched found for content => \n" + content);
- }
- } catch (IOException e) {
- throw new AiravataException("Error while reading content of the email message");
- }
- return jobStatusResult;
- }
-
- private JobState getJobState(String status, String content) {
- switch (status) {
- case "Aborted":
- return JobState.FAILED;
- case "Scucess":
- return JobState.COMPLETE;
- default:
- return JobState.UNKNOWN;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/PBSEmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/PBSEmailParser.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/PBSEmailParser.java
deleted file mode 100644
index cd6142c..0000000
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/PBSEmailParser.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.core.monitor.mail.parser;
-
-import org.apache.airavata.common.exception.AiravataException;
-import org.apache.airavata.gfac.core.monitor.mail.JobStatusResult;
-import org.apache.airavata.model.workspace.experiment.JobState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.mail.Message;
-import javax.mail.MessagingException;
-import java.io.IOException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class PBSEmailParser implements EmailParser {
-
- private static final Logger log = LoggerFactory.getLogger(PBSEmailParser.class);
-
-
- private static final String REGEX = "[a-zA-Z: ]*(?<" + JOBID + ">[a-zA-Z0-9-\\.]*)\\s+.*\\s+.*\\s+(?<"
- + STATUS + ">[a-zA-Z\\ ]*)";
- private static final String REGEX_EXIT_STATUS = "Exit_status=(?<" + EXIT_STATUS + ">[\\d]+)";
-
- @Override
- public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException {
- JobStatusResult jobStatusResult = new JobStatusResult();
- try {
- String content = ((String) message.getContent());
- Pattern pattern = Pattern.compile(REGEX);
- Matcher matcher = pattern.matcher(content);
- if (matcher.find()) {
- jobStatusResult.setJobId(matcher.group(JOBID));
- String statusLine = matcher.group(STATUS);
- jobStatusResult.setState(getJobState(statusLine, content));
- return jobStatusResult;
- } else {
- log.error("No matched found for content => \n" + content);
- }
-
- } catch (IOException e) {
- throw new AiravataException("Error while reading content of the email message");
- }
- return jobStatusResult;
- }
-
- private JobState getJobState(String statusLine, String content) {
- switch (statusLine) {
- case "Begun execution":
- return JobState.QUEUED;
- case "Execution terminated":
- int exitStatus = getExitStatus(content);
- switch (exitStatus) {
- case 0:
- return JobState.COMPLETE;
- case 1:
- return JobState.FAILED;
- default:
- return JobState.UNKNOWN;
- }
- default:
- return JobState.UNKNOWN;
- }
- }
-
- private int getExitStatus(String content) {
- Pattern pattern = Pattern.compile(REGEX_EXIT_STATUS);
- Matcher matcher = pattern.matcher(content);
- if (matcher.find()) {
- String group = matcher.group(EXIT_STATUS);
- if (group != null && !group.trim().isEmpty()) {
- return Integer.valueOf(group.trim());
- }
- }
- return -1;
- }
-
-
-
-/* -----------------------
- This is the message envelope
- ---------------------------
- FROM: pbsconsult@sdsc.edu
- TO: shameera@scigap.org
- SUBJECT: PBS JOB 2556782.trestles-fe1.local
- ----------------------------
- CONTENT-TYPE: TEXT/PLAIN
- This is plain text
- ---------------------------
- PBS Job Id: 2556782.trestles-fe1.local
- Job Name: A1182004055
- Exec host: trestles-1-12/0+trestles-1-12/1+trestles-1-12/2+trestles-1-12/3
- Begun execution
- */
-/*
- -----------------------
- This is the message envelope
- ---------------------------
- FROM: pbsconsult@sdsc.edu
- TO: shameera@scigap.org
- SUBJECT: PBS JOB 2556782.trestles-fe1.local
- ----------------------------
- CONTENT-TYPE: TEXT/PLAIN
- This is plain text
- ---------------------------
- PBS Job Id: 2556782.trestles-fe1.local
- Job Name: A1182004055
- Exec host: trestles-1-12/0+trestles-1-12/1+trestles-1-12/2+trestles-1-12/3
- Execution terminated
- Exit_status=0
- resources_used.cput=00:14:31
- resources_used.mem=124712kb
- resources_used.vmem=3504116kb
- resources_used.walltime=00:04:10
- Error_Path: trestles-login2.sdsc.edu:/oasis/scratch/trestles/ogce/temp_project/gta-work-dirs/MonitorTest_9169517d-e2d9-4ff5-bed5-dee6eb3eebb2/Amber_Sander.stderr
- Output_Path: trestles-login2.sdsc.edu:/oasis/scratch/trestles/ogce/temp_project/gta-work-dirs/MonitorTest_9169517d-e2d9-4ff5-bed5-dee6eb3eebb2/Amber_Sander.stdout
- */
-
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/SLURMEmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/SLURMEmailParser.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/SLURMEmailParser.java
deleted file mode 100644
index ceb0013..0000000
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/SLURMEmailParser.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.core.monitor.mail.parser;
-
-import org.apache.airavata.common.exception.AiravataException;
-import org.apache.airavata.gfac.core.monitor.mail.JobStatusResult;
-import org.apache.airavata.model.workspace.experiment.JobState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.mail.Message;
-import javax.mail.MessagingException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class SLURMEmailParser implements EmailParser {
-
- private static final Logger log = LoggerFactory.getLogger(SLURMEmailParser.class);
-
- private static final String JOBID = "jobId";
- private static final String STATUS = "status";
- private static final String REGEX = "[A-Z]*\\s[a-zA-Z]*_[a-z]*=(?<" + JOBID
- + ">\\d*)\\s[a-zA-Z]*=[a-zA-Z0-9-]*\\s(?<" + STATUS + ">[]a-zA-Z]*),.*";
-
- @Override
- public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException{
- JobStatusResult jobStatusResult = new JobStatusResult();
- String subject = message.getSubject();
- Pattern pattern = Pattern.compile(REGEX);
- Matcher matcher = pattern.matcher(subject);
- if (matcher.find()) {
- jobStatusResult.setJobId(matcher.group(JOBID));
- jobStatusResult.setState(getJobState(matcher.group(STATUS)));
- // TODO remove following test lines
- String jobId = matcher.group(JOBID);
- String status = matcher.group(STATUS);
- log.info("SLURM " + status + " message received -> " + jobId);
- return jobStatusResult;
- } else {
- log.error("No matched found for subject -> " + subject);
- }
- return jobStatusResult;
- }
-
- private JobState getJobState(String state) {
- switch (state.trim()) {
- case "Began":
- return JobState.QUEUED;
- case "Ended":
- return JobState.COMPLETE;
- case "Failed":
- return JobState.FAILED;
- default:
- log.error("Job State " + state + " isn't handle by SLURM parser");
- return JobState.UNKNOWN;
-
- }
- }
-
-
-
-/* -----------------------
- This is the message envelope
- ---------------------------
- FROM: slurm@batch1.stampede.tacc.utexas.edu
- TO: shameera@scigap.org
- SUBJECT: SLURM Job_id=5055468 Name=A433255759 Began, Queued time 00:00:01
- ----------------------------
- CONTENT-TYPE: TEXT/PLAIN; charset=us-ascii
- This is plain text
- ---------------------------*/
-
-/* -----------------------
- This is the message envelope
- ---------------------------
- FROM: slurm@batch1.stampede.tacc.utexas.edu
- TO: shameera@scigap.org
- SUBJECT: SLURM Job_id=5055468 Name=A433255759 Ended, Run time 00:02:40
- ----------------------------
- CONTENT-TYPE: TEXT/PLAIN; charset=us-ascii
- This is plain text
- ---------------------------*/
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-email-monitor/pom.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-email-monitor/pom.xml b/modules/gfac/gfac-monitor/gfac-email-monitor/pom.xml
new file mode 100644
index 0000000..0bff287
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-email-monitor/pom.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>airavata-gfac-monitor</artifactId>
+ <groupId>org.apache.airavata</groupId>
+ <version>0.15-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>airavata-gfac-email-monitor</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>javax.mail</groupId>
+ <artifactId>mail</artifactId>
+ <version>1.4.7</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-gfac-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-common-utils</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-data-models</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailBasedMonitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailBasedMonitor.java b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailBasedMonitor.java
new file mode 100644
index 0000000..6e12d83
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailBasedMonitor.java
@@ -0,0 +1,229 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.email;
+
+import org.apache.airavata.common.exception.AiravataException;
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.logger.AiravataLogger;
+import org.apache.airavata.common.logger.AiravataLoggerFactory;
+import org.apache.airavata.common.utils.MonitorPublisher;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.gfac.core.context.JobExecutionContext;
+import org.apache.airavata.gfac.core.utils.GFacThreadPoolExecutor;
+import org.apache.airavata.gfac.core.utils.OutHandlerWorker;
+import org.apache.airavata.gfac.monitor.email.parser.EmailParser;
+import org.apache.airavata.gfac.monitor.email.parser.LonestarEmailParser;
+import org.apache.airavata.gfac.monitor.email.parser.PBSEmailParser;
+import org.apache.airavata.gfac.monitor.email.parser.SLURMEmailParser;
+import org.apache.airavata.model.messaging.event.JobIdentifier;
+import org.apache.airavata.model.messaging.event.JobStatusChangeRequestEvent;
+import org.apache.airavata.model.workspace.experiment.JobState;
+import org.apache.airavata.model.workspace.experiment.JobStatus;
+
+import javax.mail.Address;
+import javax.mail.Flags;
+import javax.mail.Folder;
+import javax.mail.Message;
+import javax.mail.MessagingException;
+import javax.mail.Session;
+import javax.mail.Store;
+import javax.mail.search.FlagTerm;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class EmailBasedMonitor implements Runnable{
+
+ private static final AiravataLogger log = AiravataLoggerFactory.getLogger(EmailBasedMonitor.class);
+
+ private static final String PBS_CONSULT_SDSC_EDU = "pbsconsult@sdsc.edu";
+ private static final String SLURM_BATCH_STAMPEDE = "slurm@batch1.stampede.tacc.utexas.edu";
+ private static final String LONESTAR_ADDRESS = "root@c312-206.ls4.tacc.utexas.edu";
+ private static EmailBasedMonitor emailBasedMonitor;
+ private final MonitorPublisher monitorPublisher;
+
+ private Session session ;
+ private Store store;
+ private Folder emailFolder;
+ private String host, emailAddress, password, folderName, mailStoreProtocol;
+ private Properties properties;
+
+ private Map<String, JobExecutionContext> jobMonitorMap = new ConcurrentHashMap<String, JobExecutionContext>();
+
+ private EmailBasedMonitor(MonitorPublisher monitorPublisher) throws ApplicationSettingsException {
+ this.monitorPublisher = monitorPublisher;
+ init();
+ }
+
+ private void init() throws ApplicationSettingsException {
+ host = ServerSettings.getEmailBasedMonitorHost();
+ emailAddress = ServerSettings.getEmailBasedMonitorAddress();
+ password = ServerSettings.getEmailBasedMonitorPassword();
+ folderName = ServerSettings.getEmailBasedMonitorFolderName();
+ mailStoreProtocol = ServerSettings.getEmailBasedMonitorStoreProtocol();
+
+ properties = new Properties();
+ properties.put("mail.store.protocol", mailStoreProtocol);
+
+ }
+
+ public static EmailBasedMonitor getInstant(MonitorPublisher monitorPublisher) throws ApplicationSettingsException {
+ if (emailBasedMonitor == null) {
+ synchronized (EmailBasedMonitor.class) {
+ if (emailBasedMonitor == null) {
+ emailBasedMonitor = new EmailBasedMonitor(monitorPublisher);
+ Thread thread = new Thread(emailBasedMonitor);
+ thread.start();
+ }
+ }
+ }
+
+ return emailBasedMonitor;
+ }
+
+ public void addToJobMonitorMap(JobExecutionContext jobExecutionContext) {
+ addToJobMonitorMap(jobExecutionContext.getJobDetails().getJobID(), jobExecutionContext);
+ }
+
+ public void addToJobMonitorMap(String jobId, JobExecutionContext jobExecutionContext) {
+ log.info("Added Job Id : " + jobId + " to email based monitor map");
+ jobMonitorMap.put(jobId, jobExecutionContext);
+ }
+
+ private JobStatusResult parse(Message message) throws MessagingException, AiravataException {
+ Address fromAddress = message.getFrom()[0];
+ EmailParser emailParser;
+ String addressStr = fromAddress.toString();
+ switch (addressStr) {
+ case PBS_CONSULT_SDSC_EDU:
+ emailParser = new PBSEmailParser();
+ break;
+ case SLURM_BATCH_STAMPEDE:
+ emailParser = new SLURMEmailParser();
+ break;
+ case LONESTAR_ADDRESS:
+ emailParser = new LonestarEmailParser();
+ break;
+ default:
+ throw new AiravataException("Un-handle address type for email monitoring --> " + addressStr);
+ }
+ return emailParser.parseEmail(message);
+ }
+
+ @Override
+ public void run() {
+ try {
+ session = Session.getDefaultInstance(properties);
+ store = session.getStore(mailStoreProtocol);
+ store.connect(host, emailAddress, password);
+ while (!ServerSettings.isStopAllThreads()) {
+ if (!store.isConnected()) {
+ store.connect();
+ }
+ Thread.sleep(2000);
+ emailFolder = store.getFolder(folderName);
+ emailFolder.open(Folder.READ_WRITE);
+ Message[] searchMessages = emailFolder.search(new FlagTerm(new Flags(Flags.Flag.SEEN), false));
+ for (Message message : searchMessages) {
+ try {
+ JobStatusResult jobStatusResult = parse(message);
+ process(jobStatusResult);
+ } catch (AiravataException e) {
+ log.error("Error parsing email message =====================================>", e);
+ try {
+ writeEnvelopeOnError(message);
+ } catch (MessagingException e1) {
+ log.error("Error printing envelop of the email");
+ }
+
+ }
+ }
+ emailFolder.setFlags(searchMessages, new Flags(Flags.Flag.SEEN), true);
+ emailFolder.close(false);
+ }
+
+ } catch (MessagingException e) {
+ log.error("Couldn't connect to the store ", e);
+ } catch (InterruptedException e) {
+ log.error("Interrupt exception while sleep ", e);
+ } finally {
+ try {
+ store.close();
+ } catch (MessagingException e) {
+ log.error("Store close operation failed, couldn't close store", e);
+ }
+ }
+ }
+
+ private void process(JobStatusResult jobStatusResult) throws AiravataException {
+ JobExecutionContext jEC = jobMonitorMap.get(jobStatusResult.getJobId());
+ if (jEC == null) {
+ throw new AiravataException("JobExecutionContext is not found for job Id " + jobStatusResult.getJobId());
+ }
+ JobState resultState = jobStatusResult.getState();
+ jEC.getJobDetails().setJobStatus(new JobStatus(resultState));
+ if (resultState == JobState.COMPLETE) {
+ GFacThreadPoolExecutor.getFixedThreadPool().submit(new OutHandlerWorker(jEC, monitorPublisher));
+ }else if (resultState == JobState.QUEUED) {
+ // TODO - publish queued rabbitmq message
+ }else if (resultState == JobState.FAILED) {
+ // TODO - handle failed scenario
+ jobMonitorMap.remove(jobStatusResult.getJobId());
+ log.info("Job failed email received , removed job from job monitoring");
+// monitorPublisher.publish(jEC.getJobDetails().getJobStatus());
+ }
+ publishJobStatusChange(jEC);
+ }
+
+ private void publishJobStatusChange(JobExecutionContext jobExecutionContext) {
+ JobStatusChangeRequestEvent jobStatus = new JobStatusChangeRequestEvent();
+ JobIdentifier jobIdentity = new JobIdentifier(jobExecutionContext.getJobDetails().getJobID(),
+ jobExecutionContext.getTaskData().getTaskID(),
+ jobExecutionContext.getWorkflowNodeDetails().getNodeInstanceId(),
+ jobExecutionContext.getExperimentID(),
+ jobExecutionContext.getGatewayID());
+ jobStatus.setJobIdentity(jobIdentity);
+ jobStatus.setState(jobExecutionContext.getJobDetails().getJobStatus().getJobState());
+ // we have this JobStatus class to handle amqp monitoring
+ log.debugId(jobStatus.getJobIdentity().getJobId(), "Published job status change request, " +
+ "experiment {} , task {}", jobStatus.getJobIdentity().getExperimentId(),
+ jobStatus.getJobIdentity().getTaskId());
+
+ monitorPublisher.publish(jobStatus);
+ }
+
+ private void writeEnvelopeOnError(Message m) throws MessagingException {
+ Address[] a;
+ // FROM
+ if ((a = m.getFrom()) != null) {
+ for (int j = 0; j < a.length; j++)
+ log.error("FROM: " + a[j].toString());
+ }
+ // TO
+ if ((a = m.getRecipients(Message.RecipientType.TO)) != null) {
+ for (int j = 0; j < a.length; j++)
+ log.error("TO: " + a[j].toString());
+ }
+ // SUBJECT
+ if (m.getSubject() != null)
+ log.error("SUBJECT: " + m.getSubject());
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/JobStatusResult.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/JobStatusResult.java b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/JobStatusResult.java
new file mode 100644
index 0000000..9145993
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/JobStatusResult.java
@@ -0,0 +1,44 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.email;
+
+import org.apache.airavata.model.workspace.experiment.JobState;
+
+public class JobStatusResult {
+ private JobState state;
+ private String jobId;
+
+ public JobState getState() {
+ return state;
+ }
+
+ public void setState(JobState state) {
+ this.state = state;
+ }
+
+ public String getJobId() {
+ return jobId;
+ }
+
+ public void setJobId(String jobId) {
+ this.jobId = jobId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/EmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/EmailParser.java b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/EmailParser.java
new file mode 100644
index 0000000..84899f0
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/EmailParser.java
@@ -0,0 +1,35 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.email.parser;
+
+import org.apache.airavata.common.exception.AiravataException;
+import org.apache.airavata.gfac.monitor.email.JobStatusResult;
+
+import javax.mail.Message;
+import javax.mail.MessagingException;
+
+public interface EmailParser {
+ static final String STATUS = "status";
+ static final String JOBID = "jobId";
+ static final String EXIT_STATUS = "exitStatus";
+
+ JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException;
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/LonestarEmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/LonestarEmailParser.java b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/LonestarEmailParser.java
new file mode 100644
index 0000000..f02a032
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/LonestarEmailParser.java
@@ -0,0 +1,74 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.email.parser;
+
+import org.apache.airavata.common.exception.AiravataException;
+import org.apache.airavata.gfac.monitor.email.JobStatusResult;
+import org.apache.airavata.model.workspace.experiment.JobState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.mail.Message;
+import javax.mail.MessagingException;
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class LonestarEmailParser implements EmailParser {
+ private static final Logger log = LoggerFactory.getLogger(LonestarEmailParser.class);
+ //root@c312-206.ls4.tacc.utexas.edu
+ private static final String SIGNAL = "signal";
+ private static final String LONESTAR_REGEX = "Job (?<" + JOBID + ">\\d+) \\(.*\\) (?<" + STATUS
+ + ">.*)\\s[a-zA-Z =]+(?<" + EXIT_STATUS + ">\\d+)\\sSignal[ ]*=[ ]*(?<" + SIGNAL + ">[a-zA-z]*)";
+
+ @Override
+ public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException {
+ JobStatusResult jobStatusResult = new JobStatusResult();
+ try {
+ String content = ((String) message.getContent());
+ Pattern pattern = Pattern.compile(LONESTAR_REGEX);
+ Matcher matcher = pattern.matcher(content);
+ if (matcher.find()) {
+ jobStatusResult.setJobId(matcher.group(JOBID));
+ String status = matcher.group(STATUS);
+ jobStatusResult.setState(getJobState(status, content));
+ return jobStatusResult;
+ } else {
+ log.error("No matched found for content => \n" + content);
+ }
+ } catch (IOException e) {
+ throw new AiravataException("Error while reading content of the email message");
+ }
+ return jobStatusResult;
+ }
+
+ private JobState getJobState(String status, String content) {
+ switch (status) {
+ case "Aborted":
+ return JobState.FAILED;
+ case "Success":
+ return JobState.COMPLETE;
+ default:
+ return JobState.UNKNOWN;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/PBSEmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/PBSEmailParser.java b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/PBSEmailParser.java
new file mode 100644
index 0000000..b748704
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/PBSEmailParser.java
@@ -0,0 +1,139 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.email.parser;
+
+import org.apache.airavata.common.exception.AiravataException;
+import org.apache.airavata.gfac.monitor.email.JobStatusResult;
+import org.apache.airavata.model.workspace.experiment.JobState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.mail.Message;
+import javax.mail.MessagingException;
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class PBSEmailParser implements EmailParser {
+
+ private static final Logger log = LoggerFactory.getLogger(PBSEmailParser.class);
+
+
+ private static final String REGEX = "[a-zA-Z: ]*(?<" + JOBID + ">[a-zA-Z0-9-\\.]*)\\s+.*\\s+.*\\s+(?<"
+ + STATUS + ">[a-zA-Z\\ ]*)";
+ private static final String REGEX_EXIT_STATUS = "Exit_status=(?<" + EXIT_STATUS + ">[\\d]+)";
+
+ @Override
+ public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException {
+ JobStatusResult jobStatusResult = new JobStatusResult();
+ try {
+ String content = ((String) message.getContent());
+ Pattern pattern = Pattern.compile(REGEX);
+ Matcher matcher = pattern.matcher(content);
+ if (matcher.find()) {
+ jobStatusResult.setJobId(matcher.group(JOBID));
+ String statusLine = matcher.group(STATUS);
+ jobStatusResult.setState(getJobState(statusLine, content));
+ return jobStatusResult;
+ } else {
+ log.error("No matched found for content => \n" + content);
+ }
+
+ } catch (IOException e) {
+ throw new AiravataException("Error while reading content of the email message");
+ }
+ return jobStatusResult;
+ }
+
+ private JobState getJobState(String statusLine, String content) {
+ switch (statusLine) {
+ case "Begun execution":
+ return JobState.QUEUED;
+ case "Execution terminated":
+ int exitStatus = getExitStatus(content);
+ switch (exitStatus) {
+ case 0:
+ return JobState.COMPLETE;
+ case 1:
+ return JobState.FAILED;
+ default:
+ return JobState.UNKNOWN;
+ }
+ default:
+ return JobState.UNKNOWN;
+ }
+ }
+
+ private int getExitStatus(String content) {
+ Pattern pattern = Pattern.compile(REGEX_EXIT_STATUS);
+ Matcher matcher = pattern.matcher(content);
+ if (matcher.find()) {
+ String group = matcher.group(EXIT_STATUS);
+ if (group != null && !group.trim().isEmpty()) {
+ return Integer.valueOf(group.trim());
+ }
+ }
+ return -1;
+ }
+
+
+
+/* -----------------------
+ This is the message envelope
+ ---------------------------
+ FROM: pbsconsult@sdsc.edu
+ TO: shameera@scigap.org
+ SUBJECT: PBS JOB 2556782.trestles-fe1.local
+ ----------------------------
+ CONTENT-TYPE: TEXT/PLAIN
+ This is plain text
+ ---------------------------
+ PBS Job Id: 2556782.trestles-fe1.local
+ Job Name: A1182004055
+ Exec host: trestles-1-12/0+trestles-1-12/1+trestles-1-12/2+trestles-1-12/3
+ Begun execution
+ */
+/*
+ -----------------------
+ This is the message envelope
+ ---------------------------
+ FROM: pbsconsult@sdsc.edu
+ TO: shameera@scigap.org
+ SUBJECT: PBS JOB 2556782.trestles-fe1.local
+ ----------------------------
+ CONTENT-TYPE: TEXT/PLAIN
+ This is plain text
+ ---------------------------
+ PBS Job Id: 2556782.trestles-fe1.local
+ Job Name: A1182004055
+ Exec host: trestles-1-12/0+trestles-1-12/1+trestles-1-12/2+trestles-1-12/3
+ Execution terminated
+ Exit_status=0
+ resources_used.cput=00:14:31
+ resources_used.mem=124712kb
+ resources_used.vmem=3504116kb
+ resources_used.walltime=00:04:10
+ Error_Path: trestles-login2.sdsc.edu:/oasis/scratch/trestles/ogce/temp_project/gta-work-dirs/MonitorTest_9169517d-e2d9-4ff5-bed5-dee6eb3eebb2/Amber_Sander.stderr
+ Output_Path: trestles-login2.sdsc.edu:/oasis/scratch/trestles/ogce/temp_project/gta-work-dirs/MonitorTest_9169517d-e2d9-4ff5-bed5-dee6eb3eebb2/Amber_Sander.stdout
+ */
+
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/SLURMEmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/SLURMEmailParser.java b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/SLURMEmailParser.java
new file mode 100644
index 0000000..643265e
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/SLURMEmailParser.java
@@ -0,0 +1,101 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.email.parser;
+
+import org.apache.airavata.common.exception.AiravataException;
+import org.apache.airavata.gfac.monitor.email.JobStatusResult;
+import org.apache.airavata.model.workspace.experiment.JobState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.mail.Message;
+import javax.mail.MessagingException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class SLURMEmailParser implements EmailParser {
+
+ private static final Logger log = LoggerFactory.getLogger(SLURMEmailParser.class);
+
+ private static final String JOBID = "jobId";
+ private static final String STATUS = "status";
+ private static final String REGEX = "[A-Z]*\\s[a-zA-Z]*_[a-z]*=(?<" + JOBID
+ + ">\\d*)\\s[a-zA-Z]*=[a-zA-Z0-9-]*\\s(?<" + STATUS + ">[]a-zA-Z]*),.*";
+
+ @Override
+ public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException{
+ JobStatusResult jobStatusResult = new JobStatusResult();
+ String subject = message.getSubject();
+ Pattern pattern = Pattern.compile(REGEX);
+ Matcher matcher = pattern.matcher(subject);
+ if (matcher.find()) {
+ jobStatusResult.setJobId(matcher.group(JOBID));
+ jobStatusResult.setState(getJobState(matcher.group(STATUS)));
+ // TODO remove following test lines
+ String jobId = matcher.group(JOBID);
+ String status = matcher.group(STATUS);
+ log.info("SLURM " + status + " message received -> " + jobId);
+ return jobStatusResult;
+ } else {
+ log.error("No matched found for subject -> " + subject);
+ }
+ return jobStatusResult;
+ }
+
+ private JobState getJobState(String state) {
+ switch (state.trim()) {
+ case "Began":
+ return JobState.QUEUED;
+ case "Ended":
+ return JobState.COMPLETE;
+ case "Failed":
+ return JobState.FAILED;
+ default:
+ log.error("Job State " + state + " isn't handle by SLURM parser");
+ return JobState.UNKNOWN;
+
+ }
+ }
+
+
+
+/* -----------------------
+ This is the message envelope
+ ---------------------------
+ FROM: slurm@batch1.stampede.tacc.utexas.edu
+ TO: shameera@scigap.org
+ SUBJECT: SLURM Job_id=5055468 Name=A433255759 Began, Queued time 00:00:01
+ ----------------------------
+ CONTENT-TYPE: TEXT/PLAIN; charset=us-ascii
+ This is plain text
+ ---------------------------*/
+
+/* -----------------------
+ This is the message envelope
+ ---------------------------
+ FROM: slurm@batch1.stampede.tacc.utexas.edu
+ TO: shameera@scigap.org
+ SUBJECT: SLURM Job_id=5055468 Name=A433255759 Ended, Run time 00:02:40
+ ----------------------------
+ CONTENT-TYPE: TEXT/PLAIN; charset=us-ascii
+ This is plain text
+ ---------------------------*/
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/pom.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/pom.xml b/modules/gfac/gfac-monitor/gfac-hpc-monitor/pom.xml
new file mode 100644
index 0000000..a3e15a2
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/pom.xml
@@ -0,0 +1,165 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>airavata-gfac-monitor</artifactId>
+ <groupId>org.apache.airavata</groupId>
+ <version>0.15-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>airavata-gfac-hpc-monitor</artifactId>
+ <name>Airavata GFac Grid Job Monitor</name>
+ <description>The Grid related monitoring implementation</description>
+ <url>http://airavata.apache.org/</url>
+
+ <dependencies>
+ <!-- Logging -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <!-- GFAC schemas -->
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-gfac-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-gfac-gsissh</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-gfac-ssh</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-workflow-execution-context</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-registry-cpi</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-jpa-registry</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <!-- Workflow Tracking -->
+ <!--<dependency>-->
+ <!--<groupId>org.apache.airavata</groupId>-->
+ <!--<artifactId>airavata-workflow-tracking</artifactId>-->
+ <!--<version>${project.version}</version>-->
+ <!--</dependency>-->
+ <!-- Credential Store -->
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-credential-store</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <!-- Test -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <version>6.1.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-server-configuration</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-client-configuration</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- Guava -->
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>12.0</version>
+ </dependency>
+ <!-- gsi-ssh api dependencies -->
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>gsissh</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-data-models</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.jcraft</groupId>
+ <artifactId>jsch</artifactId>
+ <version>0.1.50</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.xmlbeans</groupId>
+ <artifactId>xmlbeans</artifactId>
+ <version>${xmlbeans.version}</version>
+ </dependency>
+ <!-- this is the dependency for amqp implementation -->
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.0.0</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <skip>false</skip>
+ <forkMode>always</forkMode>
+ <failIfNoTests>false</failIfNoTests>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.jsonschema2pojo</groupId>
+ <artifactId>jsonschema2pojo-maven-plugin</artifactId>
+ <version>0.4.0</version>
+ <configuration>
+ <sourceDirectory>${basedir}/src/main/resources/schema</sourceDirectory>
+ <targetPackage>org.apache.airavata</targetPackage>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>generate</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/HPCMonitorID.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/HPCMonitorID.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/HPCMonitorID.java
new file mode 100644
index 0000000..ae463a7
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/HPCMonitorID.java
@@ -0,0 +1,107 @@
+package org.apache.airavata.gfac.monitor;/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.SecurityContext;
+import org.apache.airavata.gfac.core.context.JobExecutionContext;
+import org.apache.airavata.gfac.core.monitor.MonitorID;
+import org.apache.airavata.gfac.gsissh.security.GSISecurityContext;
+import org.apache.airavata.gfac.ssh.security.SSHSecurityContext;
+import org.apache.airavata.gsi.ssh.api.ServerInfo;
+import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
+import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo;
+import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Timestamp;
+import java.util.Date;
+
+public class HPCMonitorID extends MonitorID {
+ private final static Logger logger = LoggerFactory.getLogger(HPCMonitorID.class);
+
+
+ private AuthenticationInfo authenticationInfo = null;
+
+ public HPCMonitorID(ComputeResourceDescription computeResourceDescription, String jobID, String taskID, String workflowNodeID,
+ String experimentID, String userName,String jobName) {
+ super(computeResourceDescription, jobID, taskID, workflowNodeID, experimentID, userName,jobName);
+ setComputeResourceDescription(computeResourceDescription);
+ setJobStartedTime(new Timestamp((new Date()).getTime()));
+ setUserName(userName);
+ setJobID(jobID);
+ setTaskID(taskID);
+ setExperimentID(experimentID);
+ setWorkflowNodeID(workflowNodeID);
+ }
+
+ public HPCMonitorID(AuthenticationInfo authenticationInfo, JobExecutionContext jobExecutionContext) {
+ super(jobExecutionContext);
+ this.authenticationInfo = authenticationInfo;
+ if (this.authenticationInfo != null) {
+ try {
+ String hostAddress = jobExecutionContext.getHostName();
+ SecurityContext securityContext = jobExecutionContext.getSecurityContext(hostAddress);
+ ServerInfo serverInfo = null;
+ if (securityContext != null) {
+ if (securityContext instanceof GSISecurityContext){
+ serverInfo = (((GSISecurityContext) securityContext).getPbsCluster()).getServerInfo();
+ if (serverInfo.getUserName() != null) {
+ setUserName(serverInfo.getUserName());
+ }
+ }
+ if (securityContext instanceof SSHSecurityContext){
+ serverInfo = (((SSHSecurityContext) securityContext).getPbsCluster()).getServerInfo();
+ if (serverInfo.getUserName() != null) {
+ setUserName(serverInfo.getUserName());
+ }
+ }
+ }
+ } catch (GFacException e) {
+ logger.error("Error while getting security context", e);
+ }
+ }
+ }
+
+ public HPCMonitorID(ComputeResourceDescription computeResourceDescription, String jobID, String taskID, String workflowNodeID, String experimentID, String userName, AuthenticationInfo authenticationInfo) {
+ setComputeResourceDescription(computeResourceDescription);
+ setJobStartedTime(new Timestamp((new Date()).getTime()));
+ this.authenticationInfo = authenticationInfo;
+ // if we give myproxyauthenticationInfo, so we try to use myproxy user as the user
+ if (this.authenticationInfo != null) {
+ if (this.authenticationInfo instanceof MyProxyAuthenticationInfo) {
+ setUserName(((MyProxyAuthenticationInfo) this.authenticationInfo).getUserName());
+ }
+ }
+ setJobID(jobID);
+ setTaskID(taskID);
+ setExperimentID(experimentID);
+ setWorkflowNodeID(workflowNodeID);
+ }
+
+ public AuthenticationInfo getAuthenticationInfo() {
+ return authenticationInfo;
+ }
+
+ public void setAuthenticationInfo(AuthenticationInfo authenticationInfo) {
+ this.authenticationInfo = authenticationInfo;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/HostMonitorData.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/HostMonitorData.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/HostMonitorData.java
new file mode 100644
index 0000000..f29e3e6
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/HostMonitorData.java
@@ -0,0 +1,88 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor;
+
+import org.apache.airavata.gfac.core.context.JobExecutionContext;
+import org.apache.airavata.gfac.core.monitor.MonitorID;
+import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
+import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
+import org.apache.airavata.model.appcatalog.computeresource.DataMovementProtocol;
+import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class HostMonitorData {
+// private HostDescription host;
+ private ComputeResourceDescription computeResourceDescription;
+ private JobSubmissionProtocol jobSubmissionProtocol;
+ private DataMovementProtocol dataMovementProtocol;
+
+ private List<MonitorID> monitorIDs;
+
+ public HostMonitorData(JobExecutionContext jobExecutionContext) {
+ this.computeResourceDescription = jobExecutionContext.getApplicationContext().getComputeResourceDescription();
+ this.jobSubmissionProtocol = jobExecutionContext.getPreferredJobSubmissionProtocol();
+ this.dataMovementProtocol = jobExecutionContext.getPreferredDataMovementProtocol();
+ this.monitorIDs = new ArrayList<MonitorID>();
+ }
+
+ public HostMonitorData(JobExecutionContext jobExecutionContext, List<MonitorID> monitorIDs) {
+ this.computeResourceDescription = jobExecutionContext.getApplicationContext().getComputeResourceDescription();
+ this.jobSubmissionProtocol = jobExecutionContext.getPreferredJobSubmissionProtocol();
+ this.dataMovementProtocol = jobExecutionContext.getPreferredDataMovementProtocol();
+ this.monitorIDs = monitorIDs;
+ }
+
+ public ComputeResourceDescription getComputeResourceDescription() {
+ return computeResourceDescription;
+ }
+
+ public void setComputeResourceDescription(ComputeResourceDescription computeResourceDescription) {
+ this.computeResourceDescription = computeResourceDescription;
+ }
+
+ public List<MonitorID> getMonitorIDs() {
+ return monitorIDs;
+ }
+
+ public void setMonitorIDs(List<MonitorID> monitorIDs) {
+ this.monitorIDs = monitorIDs;
+ }
+
+ /**
+ * this method get called by CommonUtils and it will check the right place before adding
+ * so there will not be a mismatch between this.host and monitorID.host
+ * @param monitorID
+ * @throws org.apache.airavata.gfac.monitor.exception.AiravataMonitorException
+ */
+ public void addMonitorIDForHost(MonitorID monitorID)throws AiravataMonitorException {
+ monitorIDs.add(monitorID);
+ }
+
+ public JobSubmissionProtocol getJobSubmissionProtocol() {
+ return jobSubmissionProtocol;
+ }
+
+ public DataMovementProtocol getDataMovementProtocol() {
+ return dataMovementProtocol;
+ }
+}
[6/7] airavata git commit: Add new gfac email monitor module to gfac
monitor module and Intorduced gfac-hpc-monitor module to keep all hoc-monitor
code
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/UserMonitorData.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/UserMonitorData.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/UserMonitorData.java
new file mode 100644
index 0000000..022d17c
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/UserMonitorData.java
@@ -0,0 +1,76 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor;
+
+import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * This is the datastructure to keep the user centric job data, rather keeping
+ * the individual jobs we keep the jobs based on the each user
+ */
+public class UserMonitorData {
+ private final static Logger logger = LoggerFactory.getLogger(UserMonitorData.class);
+
+ private String userName;
+
+ private List<HostMonitorData> hostMonitorData;
+
+
+ public UserMonitorData(String userName) {
+ this.userName = userName;
+ hostMonitorData = new ArrayList<HostMonitorData>();
+ }
+
+ public UserMonitorData(String userName, List<HostMonitorData> hostMonitorDataList) {
+ this.hostMonitorData = hostMonitorDataList;
+ this.userName = userName;
+ }
+
+ public List<HostMonitorData> getHostMonitorData() {
+ return hostMonitorData;
+ }
+
+ public void setHostMonitorData(List<HostMonitorData> hostMonitorData) {
+ this.hostMonitorData = hostMonitorData;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ /*
+ This method will add element to the MonitorID list, user should not
+ duplicate it, we do not check it because its going to be used by airavata
+ so we have to use carefully and this method will add a host if its a new host
+ */
+ public void addHostMonitorData(HostMonitorData hostMonitorData) throws AiravataMonitorException {
+ this.hostMonitorData.add(hostMonitorData);
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/command/ExperimentCancelRequest.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/command/ExperimentCancelRequest.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/command/ExperimentCancelRequest.java
new file mode 100644
index 0000000..f19decf
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/command/ExperimentCancelRequest.java
@@ -0,0 +1,38 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.monitor.command;
+
+public class ExperimentCancelRequest {
+ private String experimentId;
+
+ public ExperimentCancelRequest(String experimentId) {
+ this.experimentId = experimentId;
+ }
+
+ public String getExperimentId() {
+ return experimentId;
+ }
+
+ public void setExperimentId(String experimentId) {
+ this.experimentId = experimentId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/command/TaskCancelRequest.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/command/TaskCancelRequest.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/command/TaskCancelRequest.java
new file mode 100644
index 0000000..b45e01c
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/command/TaskCancelRequest.java
@@ -0,0 +1,52 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.monitor.command;
+
+public class TaskCancelRequest {
+ private String experimentId;
+ private String nodeId;
+ private String taskId;
+
+ public TaskCancelRequest(String experimentId, String nodeId, String taskId) {
+ this.experimentId = experimentId;
+ this.setNodeId(nodeId);
+ this.taskId = taskId;
+ }
+ public String getExperimentId() {
+ return experimentId;
+ }
+ public void setExperimentId(String experimentId) {
+ this.experimentId = experimentId;
+ }
+ public String getTaskId() {
+ return taskId;
+ }
+ public void setTaskId(String taskId) {
+ this.taskId = taskId;
+ }
+ public String getNodeId() {
+ return nodeId;
+ }
+ public void setNodeId(String nodeId) {
+ this.nodeId = nodeId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/AiravataAbstractMonitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/AiravataAbstractMonitor.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/AiravataAbstractMonitor.java
new file mode 100644
index 0000000..c754d3c
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/AiravataAbstractMonitor.java
@@ -0,0 +1,46 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.core;
+
+import org.apache.airavata.common.utils.MonitorPublisher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This is the abstract Monitor which needs to be used by
+ * any Monitoring implementation which expect nto consume
+ * to store the status to registry. Because they have to
+ * use the MonitorPublisher to publish the monitoring statuses
+ * to the Event Bus. All the Monitor statuses publish to the eventbus
+ * will be saved to the Registry.
+ */
+public abstract class AiravataAbstractMonitor implements Monitor {
+ private final static Logger logger = LoggerFactory.getLogger(AiravataAbstractMonitor.class);
+ protected MonitorPublisher publisher;
+
+ public MonitorPublisher getPublisher() {
+ return publisher;
+ }
+
+ public void setPublisher(MonitorPublisher publisher) {
+ this.publisher = publisher;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/MessageParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/MessageParser.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/MessageParser.java
new file mode 100644
index 0000000..a003f55
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/MessageParser.java
@@ -0,0 +1,43 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.core;
+
+import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
+import org.apache.airavata.model.workspace.experiment.JobState;
+
+/**
+ * This is an interface to implement messageparser, it could be
+ * pull based or push based still monitor has to parse the content of
+ * the message it gets from remote monitoring system and finalize
+ * them to internal job state, Ex: JSON parser for AMQP and Qstat reader
+ * for pull based monitor.
+ */
+public interface MessageParser {
+ /**
+ * This method is to implement how to parse the incoming message
+ * and implement a logic to finalize the status of the job,
+ * we have to makesure the correct message is given to the messageparser
+ * parse method, it will not do any filtering
+ * @param message content of the message
+ * @return
+ */
+ JobState parseMessage(String message)throws AiravataMonitorException;
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/Monitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/Monitor.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/Monitor.java
new file mode 100644
index 0000000..614d606
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/Monitor.java
@@ -0,0 +1,30 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.core;
+
+
+/**
+ * This is the primary interface for Monitors,
+ * This can be used to implement different methods of monitoring
+ */
+public interface Monitor {
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/PullMonitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/PullMonitor.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/PullMonitor.java
new file mode 100644
index 0000000..efdf89c
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/PullMonitor.java
@@ -0,0 +1,64 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.core;
+
+import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
+
+/**
+ * PullMonitors can implement this interface
+ * Since the pull and push based monitoring required different
+ * operations, PullMonitor will be useful.
+ * This will allow users to program Pull monitors separately
+ */
+public abstract class PullMonitor extends AiravataAbstractMonitor {
+
+ private int pollingFrequence;
+ /**
+ * This method will can invoke when PullMonitor needs to start
+ * and it has to invoke in the frequency specified below,
+ * @return if the start process is successful return true else false
+ */
+ public abstract boolean startPulling() throws AiravataMonitorException;
+
+ /**
+ * This is the method to stop the polling process
+ * @return if the stopping process is successful return true else false
+ */
+ public abstract boolean stopPulling()throws AiravataMonitorException;
+
+ /**
+ * this method can be used to set the polling frequencey or otherwise
+ * can implement a polling mechanism, and implement how to do
+ * @param frequence
+ */
+ public void setPollingFrequence(int frequence){
+ this.pollingFrequence = frequence;
+ }
+
+ /**
+ * this method can be used to get the polling frequencey or otherwise
+ * can implement a polling mechanism, and implement how to do
+ * @return
+ */
+ public int getPollingFrequence(){
+ return this.pollingFrequence;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/PushMonitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/PushMonitor.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/PushMonitor.java
new file mode 100644
index 0000000..1b6a228
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/PushMonitor.java
@@ -0,0 +1,60 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.core;
+
+import org.apache.airavata.gfac.core.monitor.MonitorID;
+import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
+
+/**
+ * PushMonitors can implement this interface
+ * Since the pull and push based monitoring required different
+ * operations, PullMonitor will be useful.
+ * This interface will allow users to program Push monitors separately
+ */
+public abstract class PushMonitor extends AiravataAbstractMonitor {
+ /**
+ * This method can be invoked to register a listener with the
+ * remote monitoring system, ideally inside this method users will be
+ * writing some client listener code for the remote monitoring system,
+ * this will be a simple wrapper around any client for the remote Monitor.
+ * @param monitorID
+ * @return
+ */
+ public abstract boolean registerListener(MonitorID monitorID)throws AiravataMonitorException;
+
+ /**
+ * This method can be invoked to unregister a listener with the
+ * remote monitoring system, ideally inside this method users will be
+ * writing some client listener code for the remote monitoring system,
+ * this will be a simple wrapper around any client for the remote Monitor.
+ * @param monitorID
+ * @return
+ */
+ public abstract boolean unRegisterListener(MonitorID monitorID)throws AiravataMonitorException;
+
+ /**
+ * This can be used to stop the registration thread
+ * @return
+ * @throws org.apache.airavata.gfac.monitor.exception.AiravataMonitorException
+ */
+ public abstract boolean stopRegister()throws AiravataMonitorException;
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/exception/AiravataMonitorException.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/exception/AiravataMonitorException.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/exception/AiravataMonitorException.java
new file mode 100644
index 0000000..3acef66
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/exception/AiravataMonitorException.java
@@ -0,0 +1,37 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.exception;
+
+public class AiravataMonitorException extends Exception {
+ private static final long serialVersionUID = -2849422320139467602L;
+
+ public AiravataMonitorException(Throwable e) {
+ super(e);
+ }
+
+ public AiravataMonitorException(String message) {
+ super(message, null);
+ }
+
+ public AiravataMonitorException(String message, Throwable e) {
+ super(message, e);
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java
new file mode 100644
index 0000000..24b300e
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java
@@ -0,0 +1,142 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.handlers;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.logger.AiravataLogger;
+import org.apache.airavata.common.logger.AiravataLoggerFactory;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.gfac.core.context.JobExecutionContext;
+import org.apache.airavata.gfac.core.cpi.BetterGfacImpl;
+import org.apache.airavata.gfac.core.handler.GFacHandlerException;
+import org.apache.airavata.gfac.core.handler.ThreadedHandler;
+import org.apache.airavata.gfac.core.monitor.MonitorID;
+import org.apache.airavata.gfac.core.utils.GFacUtils;
+import org.apache.airavata.gfac.monitor.HPCMonitorID;
+import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
+import org.apache.airavata.gfac.monitor.impl.pull.qstat.HPCPullMonitor;
+import org.apache.airavata.gfac.monitor.util.CommonUtils;
+import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
+import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.data.Stat;
+
+import java.io.File;
+import java.util.Properties;
+
+/**
+ * this handler is responsible for monitoring jobs in a pull mode
+ * and currently this support multiple pull monitoring in grid resource and uses
+ * commands like qstat,squeue and this supports sun grid enging monitoring too
+ * which is a slight variation of qstat monitoring.
+ */
+public class GridPullMonitorHandler extends ThreadedHandler implements Watcher{
+ private final static AiravataLogger logger = AiravataLoggerFactory.getLogger(GridPullMonitorHandler.class);
+
+ private HPCPullMonitor hpcPullMonitor;
+
+ private AuthenticationInfo authenticationInfo;
+
+ public void initProperties(Properties properties) throws GFacHandlerException {
+ String myProxyUser = null;
+ try {
+ myProxyUser = ServerSettings.getSetting("myproxy.username");
+ String myProxyPass = ServerSettings.getSetting("myproxy.password");
+ String certPath = ServerSettings.getSetting("trusted.cert.location");
+ String myProxyServer = ServerSettings.getSetting("myproxy.server");
+ setAuthenticationInfo(new MyProxyAuthenticationInfo(myProxyUser, myProxyPass, myProxyServer,
+ 7512, 17280000, certPath));
+ if(BetterGfacImpl.getMonitorPublisher() != null){
+ hpcPullMonitor = new HPCPullMonitor(BetterGfacImpl.getMonitorPublisher(),getAuthenticationInfo()); // we use our own credentials for monitoring, not from the store
+ }else {
+ throw new GFacHandlerException("Error initializing Monitor Handler, because Monitor Publisher is null !!!");
+ }
+ } catch (ApplicationSettingsException e) {
+ logger.error("Error while reading server properties", e);
+ throw new GFacHandlerException("Error while reading server properties", e);
+ }
+ }
+
+ public void run() {
+ hpcPullMonitor.run();
+ }
+
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ super.invoke(jobExecutionContext);
+ hpcPullMonitor.setGfac(jobExecutionContext.getGfac());
+ MonitorID monitorID = new HPCMonitorID(getAuthenticationInfo(), jobExecutionContext);
+ try {
+ ZooKeeper zk = jobExecutionContext.getZk();
+ try {
+ String experimentEntry = GFacUtils.findExperimentEntry(jobExecutionContext.getExperimentID(), zk);
+ String path = experimentEntry + File.separator + "operation";
+ Stat exists = zk.exists(path, this);
+ if (exists != null) {
+ zk.getData(path, this, exists); // watching the operations node
+ }
+ } catch (KeeperException e) {
+ logger.error(e.getMessage(), e);
+ } catch (InterruptedException e) {
+ logger.error(e.getMessage(), e);
+ }
+ CommonUtils.addMonitortoQueue(hpcPullMonitor.getQueue(), monitorID, jobExecutionContext);
+ CommonUtils.increaseZkJobCount(monitorID); // update change job count to zookeeper
+ } catch (AiravataMonitorException e) {
+ logger.errorId(monitorID.getJobID(), "Error adding job {} monitorID object to the queue with experiment {}",
+ monitorID.getJobID(), monitorID.getExperimentID());
+ }
+ }
+ public AuthenticationInfo getAuthenticationInfo() {
+ return authenticationInfo;
+ }
+
+ public HPCPullMonitor getHpcPullMonitor() {
+ return hpcPullMonitor;
+ }
+
+ public void setAuthenticationInfo(AuthenticationInfo authenticationInfo) {
+ this.authenticationInfo = authenticationInfo;
+ }
+
+ public void setHpcPullMonitor(HPCPullMonitor hpcPullMonitor) {
+ this.hpcPullMonitor = hpcPullMonitor;
+ }
+
+
+ public void process(WatchedEvent watchedEvent) {
+ logger.info(watchedEvent.getPath());
+ if(Event.EventType.NodeDataChanged.equals(watchedEvent.getType())){
+ // node data is changed, this means node is cancelled.
+ logger.info("Experiment is cancelled with this path:"+watchedEvent.getPath());
+
+ String[] split = watchedEvent.getPath().split("/");
+ for(String element:split) {
+ if (element.contains("+")) {
+ logger.info("Adding experimentID+TaskID to be removed from monitoring:"+element);
+ hpcPullMonitor.getCancelJobList().add(element);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPushMonitorHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPushMonitorHandler.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPushMonitorHandler.java
new file mode 100644
index 0000000..0eb4526
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPushMonitorHandler.java
@@ -0,0 +1,103 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.handlers;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.gfac.core.context.JobExecutionContext;
+import org.apache.airavata.gfac.core.cpi.BetterGfacImpl;
+import org.apache.airavata.gfac.core.handler.GFacHandlerException;
+import org.apache.airavata.gfac.core.handler.ThreadedHandler;
+import org.apache.airavata.gfac.core.monitor.MonitorID;
+import org.apache.airavata.gfac.monitor.HPCMonitorID;
+import org.apache.airavata.gfac.monitor.impl.push.amqp.AMQPMonitor;
+import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
+import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * this handler is responsible monitoring jobs in push mode
+ * and currently this support multiple push monitoring in grid resource
+ */
+public class GridPushMonitorHandler extends ThreadedHandler {
+ private final static Logger logger= LoggerFactory.getLogger(GridPushMonitorHandler.class);
+
+ private AMQPMonitor amqpMonitor;
+
+ private AuthenticationInfo authenticationInfo;
+
+ @Override
+ public void initProperties(Properties properties) throws GFacHandlerException {
+ String myProxyUser=null;
+ try{
+ myProxyUser = ServerSettings.getSetting("myproxy.username");
+ String myProxyPass = ServerSettings.getSetting("myproxy.password");
+ String certPath = ServerSettings.getSetting("trusted.cert.location");
+ String myProxyServer = ServerSettings.getSetting("myproxy.server");
+ setAuthenticationInfo(new MyProxyAuthenticationInfo(myProxyUser, myProxyPass, myProxyServer,
+ 7512, 17280000, certPath));
+
+ String hostList=(String)properties.get("hosts");
+ String proxyFilePath = ServerSettings.getSetting("proxy.file.path");
+ String connectionName=ServerSettings.getSetting("connection.name");
+ LinkedBlockingQueue<MonitorID> pushQueue = new LinkedBlockingQueue<MonitorID>();
+ LinkedBlockingQueue<MonitorID> finishQueue = new LinkedBlockingQueue<MonitorID>();
+ List<String> hosts= Arrays.asList(hostList.split(","));
+ amqpMonitor=new AMQPMonitor(BetterGfacImpl.getMonitorPublisher(),pushQueue,finishQueue,proxyFilePath,connectionName,hosts);
+ }catch (ApplicationSettingsException e){
+ logger.error(e.getMessage(), e);
+ throw new GFacHandlerException(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public void run() {
+ amqpMonitor.run();
+ }
+
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException{
+ super.invoke(jobExecutionContext);
+ MonitorID monitorID=new HPCMonitorID(getAuthenticationInfo(),jobExecutionContext);
+ amqpMonitor.getRunningQueue().add(monitorID);
+ }
+
+ public AMQPMonitor getAmqpMonitor() {
+ return amqpMonitor;
+ }
+
+ public void setAmqpMonitor(AMQPMonitor amqpMonitor) {
+ this.amqpMonitor = amqpMonitor;
+ }
+
+ public AuthenticationInfo getAuthenticationInfo() {
+ return authenticationInfo;
+ }
+
+ public void setAuthenticationInfo(AuthenticationInfo authenticationInfo) {
+ this.authenticationInfo = authenticationInfo;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
new file mode 100644
index 0000000..ab934ca
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
@@ -0,0 +1,469 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.impl.pull.qstat;
+
+import com.google.common.eventbus.EventBus;
+import org.apache.airavata.common.logger.AiravataLogger;
+import org.apache.airavata.common.logger.AiravataLoggerFactory;
+import org.apache.airavata.common.utils.MonitorPublisher;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.gfac.monitor.util.CommonUtils;
+import org.apache.airavata.gfac.core.cpi.GFac;
+import org.apache.airavata.gfac.core.monitor.MonitorID;
+import org.apache.airavata.gfac.core.utils.GFacThreadPoolExecutor;
+import org.apache.airavata.gfac.core.utils.OutHandlerWorker;
+import org.apache.airavata.gfac.monitor.HostMonitorData;
+import org.apache.airavata.gfac.monitor.UserMonitorData;
+import org.apache.airavata.gfac.monitor.core.PullMonitor;
+import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
+import org.apache.airavata.gfac.monitor.impl.push.amqp.SimpleJobFinishConsumer;
+import org.apache.airavata.gsi.ssh.api.SSHApiException;
+import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
+import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
+import org.apache.airavata.model.messaging.event.JobIdentifier;
+import org.apache.airavata.model.messaging.event.JobStatusChangeRequestEvent;
+import org.apache.airavata.model.workspace.experiment.JobState;
+
+import java.sql.Timestamp;
+import java.util.*;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.LinkedBlockingQueue;
+
+/**
+ * This monitor is based on qstat command which can be run
+ * in grid resources and retrieve the job status.
+ */
+public class HPCPullMonitor extends PullMonitor {
+
+ private final static AiravataLogger logger = AiravataLoggerFactory.getLogger(HPCPullMonitor.class);
+ public static final int FAILED_COUNT = 5;
+
+ // I think this should use DelayedBlocking Queue to do the monitoring*/
+ private BlockingQueue<UserMonitorData> queue;
+
+ private boolean startPulling = false;
+
+ private Map<String, ResourceConnection> connections;
+
+ private MonitorPublisher publisher;
+
+ private LinkedBlockingQueue<String> cancelJobList;
+
+ private List<String> completedJobsFromPush;
+
+ private GFac gfac;
+
+ private AuthenticationInfo authenticationInfo;
+
+ private ArrayList<MonitorID> removeList;
+
+ public HPCPullMonitor() {
+ connections = new HashMap<String, ResourceConnection>();
+ queue = new LinkedBlockingDeque<UserMonitorData>();
+ publisher = new MonitorPublisher(new EventBus());
+ cancelJobList = new LinkedBlockingQueue<String>();
+ completedJobsFromPush = new ArrayList<String>();
+ (new SimpleJobFinishConsumer(this.completedJobsFromPush)).listen();
+ removeList = new ArrayList<MonitorID>();
+ }
+
+ public HPCPullMonitor(MonitorPublisher monitorPublisher, AuthenticationInfo authInfo) {
+ connections = new HashMap<String, ResourceConnection>();
+ queue = new LinkedBlockingDeque<UserMonitorData>();
+ publisher = monitorPublisher;
+ authenticationInfo = authInfo;
+ cancelJobList = new LinkedBlockingQueue<String>();
+ this.completedJobsFromPush = new ArrayList<String>();
+ (new SimpleJobFinishConsumer(this.completedJobsFromPush)).listen();
+ removeList = new ArrayList<MonitorID>();
+ }
+
+ public HPCPullMonitor(BlockingQueue<UserMonitorData> queue, MonitorPublisher publisher) {
+ this.queue = queue;
+ this.publisher = publisher;
+ connections = new HashMap<String, ResourceConnection>();
+ cancelJobList = new LinkedBlockingQueue<String>();
+ this.completedJobsFromPush = new ArrayList<String>();
+ (new SimpleJobFinishConsumer(this.completedJobsFromPush)).listen();
+ removeList = new ArrayList<MonitorID>();
+ }
+
+
+ public void run() {
+ /* implement a logic to pick each monitorID object from the queue and do the
+ monitoring
+ */
+ this.startPulling = true;
+ while (this.startPulling && !ServerSettings.isStopAllThreads()) {
+ try {
+ // After finishing one iteration of the full queue this thread sleeps 1 second
+ synchronized (this.queue) {
+ if (this.queue.size() > 0) {
+ startPulling();
+ }
+ }
+ Thread.sleep(10000);
+ } catch (Exception e) {
+ // we catch all the exceptions here because no matter what happens we do not stop running this
+ // thread, but ideally we should report proper error messages, but this is handled in startPulling
+ // method, incase something happen in Thread.sleep we handle it with this catch block.
+ logger.error(e.getMessage(),e);
+ }
+ }
+ // thread is going to return so we close all the connections
+ Iterator<String> iterator = connections.keySet().iterator();
+ while (iterator.hasNext()) {
+ String next = iterator.next();
+ ResourceConnection resourceConnection = connections.get(next);
+ try {
+ resourceConnection.getCluster().disconnect();
+ } catch (SSHApiException e) {
+ logger.error("Erro while connecting to the cluster", e);
+ }
+ }
+ }
+
+ /**
+ * This method will can invoke when PullMonitor needs to start
+ * and it has to invoke in the frequency specified below,
+ *
+ * @return if the start process is successful return true else false
+ */
+ public boolean startPulling() throws AiravataMonitorException {
+ // take the top element in the queue and pull the data and put that element
+ // at the tail of the queue
+ //todo this polling will not work with multiple usernames but with single user
+ // and multiple hosts, currently monitoring will work
+ UserMonitorData take = null;
+ JobStatusChangeRequestEvent jobStatus = new JobStatusChangeRequestEvent();
+ MonitorID currentMonitorID = null;
+ try {
+ take = this.queue.take();
+ List<HostMonitorData> hostMonitorData = take.getHostMonitorData();
+ for (ListIterator<HostMonitorData> hostIterator = hostMonitorData.listIterator(); hostIterator.hasNext();) {
+ HostMonitorData iHostMonitorData = hostIterator.next();
+ if (iHostMonitorData.getJobSubmissionProtocol() == JobSubmissionProtocol.SSH) {
+ String hostName = iHostMonitorData.getComputeResourceDescription().getHostName();
+ ResourceConnection connection = null;
+ if (connections.containsKey(hostName)) {
+ if (!connections.get(hostName).isConnected()) {
+ connection = new ResourceConnection(iHostMonitorData, getAuthenticationInfo());
+ connections.put(hostName, connection);
+ } else {
+ logger.debug("We already have this connection so not going to create one");
+ connection = connections.get(hostName);
+ }
+ } else {
+ connection = new ResourceConnection(iHostMonitorData, getAuthenticationInfo());
+ connections.put(hostName, connection);
+ }
+
+ // before we get the statuses, we check the cancel job list and remove them permanently
+ List<MonitorID> monitorID = iHostMonitorData.getMonitorIDs();
+ Iterator<String> iterator1 = cancelJobList.iterator();
+ ListIterator<MonitorID> monitorIDListIterator = monitorID.listIterator();
+ while (monitorIDListIterator.hasNext()) {
+ MonitorID iMonitorID = monitorIDListIterator.next();
+ while (iterator1.hasNext()) {
+ String cancelMId = iterator1.next();
+ if (cancelMId.equals(iMonitorID.getExperimentID() + "+" + iMonitorID.getTaskID())) {
+ iMonitorID.setStatus(JobState.CANCELED);
+// CommonUtils.removeMonitorFromQueue(take, iMonitorID);
+ removeList.add(iMonitorID);
+ logger.debugId(cancelMId, "Found a match in cancel monitor queue, hence moved to the " +
+ "completed job queue, experiment {}, task {} , job {}",
+ iMonitorID.getExperimentID(), iMonitorID.getTaskID(), iMonitorID.getJobID());
+ logger.info("Job cancelled: marking the Job as ************CANCELLED************ experiment {}, task {}, job name {} .",
+ iMonitorID.getExperimentID(), iMonitorID.getTaskID(), iMonitorID.getJobName());
+ sendNotification(iMonitorID);
+ logger.info("To avoid timing issues we sleep sometime and try to retrieve output files");
+ Thread.sleep(10000);
+ GFacThreadPoolExecutor.getFixedThreadPool().submit(new OutHandlerWorker(gfac, iMonitorID, publisher));
+ break;
+ }
+ }
+ iterator1 = cancelJobList.iterator();
+ }
+
+ cleanup(take);
+
+ synchronized (completedJobsFromPush) {
+ for (ListIterator<String> iterator = completedJobsFromPush.listIterator(); iterator.hasNext(); ) {
+ String completeId = iterator.next();
+ for (monitorIDListIterator = monitorID.listIterator(); monitorIDListIterator.hasNext(); ) {
+ MonitorID iMonitorID = monitorIDListIterator.next();
+ if (completeId.equals(iMonitorID.getUserName() + "," + iMonitorID.getJobName())) {
+ logger.info("This job is finished because push notification came with <username,jobName> " + completeId);
+ iMonitorID.setStatus(JobState.COMPLETE);
+// CommonUtils.removeMonitorFromQueue(take, iMonitorID);//we have to make this empty everytime we iterate, otherwise this list will accumulate and will lead to a memory leak
+ removeList.add(iMonitorID);
+ logger.debugId(completeId, "Push notification updated job {} status to {}. " +
+ "experiment {} , task {}.", iMonitorID.getJobID(), JobState.COMPLETE.toString(),
+ iMonitorID.getExperimentID(), iMonitorID.getTaskID());
+ logger.info("AMQP message recieved: marking the Job as ************COMPLETE************ experiment {}, task {}, job name {} .",
+ iMonitorID.getExperimentID(), iMonitorID.getTaskID(), iMonitorID.getJobName());
+
+ sendNotification(iMonitorID);
+ GFacThreadPoolExecutor.getFixedThreadPool().submit(new OutHandlerWorker(gfac, iMonitorID, publisher));
+ break;
+ }
+ }
+ }
+ }
+
+ cleanup(take);
+
+ // we have to get this again because we removed the already completed jobs with amqp messages
+ monitorID = iHostMonitorData.getMonitorIDs();
+ Map<String, JobState> jobStatuses = connection.getJobStatuses(monitorID);
+ for (Iterator<MonitorID> iterator = monitorID.listIterator(); iterator.hasNext(); ) {
+ MonitorID iMonitorID = iterator.next();
+ currentMonitorID = iMonitorID;
+ if (!JobState.CANCELED.equals(iMonitorID.getStatus()) &&
+ !JobState.COMPLETE.equals(iMonitorID.getStatus())) {
+ iMonitorID.setStatus(jobStatuses.get(iMonitorID.getJobID() + "," + iMonitorID.getJobName())); //IMPORTANT this is NOT a simple setter we have a logic
+ } else if (JobState.COMPLETE.equals(iMonitorID.getStatus())) {
+ logger.debugId(iMonitorID.getJobID(), "Moved job {} to completed jobs map, experiment {}, " +
+ "task {}", iMonitorID.getJobID(), iMonitorID.getExperimentID(), iMonitorID.getTaskID());
+// CommonUtils.removeMonitorFromQueue(take, iMonitorID);
+ removeList.add(iMonitorID);
+ logger.info("PULL Notification is complete: marking the Job as ************COMPLETE************ experiment {}, task {}, job name {} .",
+ iMonitorID.getExperimentID(), iMonitorID.getTaskID(), iMonitorID.getJobName());
+ GFacThreadPoolExecutor.getFixedThreadPool().submit(new OutHandlerWorker(gfac, iMonitorID, publisher));
+ }
+ iMonitorID.setStatus(jobStatuses.get(iMonitorID.getJobID() + "," + iMonitorID.getJobName())); //IMPORTANT this is not a simple setter we have a logic
+ iMonitorID.setLastMonitored(new Timestamp((new Date()).getTime()));
+ sendNotification(iMonitorID);
+ // if the job is completed we do not have to put the job to the queue again
+ iMonitorID.setLastMonitored(new Timestamp((new Date()).getTime()));
+ }
+
+ cleanup(take);
+
+
+ for (Iterator<MonitorID> iterator = monitorID.listIterator(); iterator.hasNext(); ) {
+ MonitorID iMonitorID = iterator.next();
+ if (iMonitorID.getFailedCount() > FAILED_COUNT) {
+ iMonitorID.setLastMonitored(new Timestamp((new Date()).getTime()));
+ String outputDir = iMonitorID.getJobExecutionContext().getOutputDir();
+ List<String> stdOut = null;
+ try {
+ stdOut = connection.getCluster().listDirectory(outputDir); // check the outputs directory
+ } catch (SSHApiException e) {
+ if (e.getMessage().contains("No such file or directory")) {
+ // this is because while we run output handler something failed and during exception
+ // we store all the jobs in the monitor queue again
+ logger.error("We know this job is already attempted to run out-handlers");
+// CommonUtils.removeMonitorFromQueue(queue, iMonitorID);
+ }
+ }
+ if (stdOut != null && stdOut.size() > 0 && !stdOut.get(0).isEmpty()) { // have to be careful with this
+ iMonitorID.setStatus(JobState.COMPLETE);
+ logger.errorId(iMonitorID.getJobID(), "Job monitoring failed {} times, " +
+ " Experiment {} , task {}", iMonitorID.getFailedCount(),
+ iMonitorID.getExperimentID(), iMonitorID.getTaskID());
+ logger.info("Listing directory came as complete: marking the Job as ************COMPLETE************ experiment {}, task {}, job name {} .",
+ iMonitorID.getExperimentID(), iMonitorID.getTaskID(), iMonitorID.getJobName());
+ sendNotification(iMonitorID);
+// CommonUtils.removeMonitorFromQueue(take, iMonitorID);
+ removeList.add(iMonitorID);
+ GFacThreadPoolExecutor.getFixedThreadPool().submit(new OutHandlerWorker(gfac, iMonitorID, publisher));
+ } else {
+ iMonitorID.setFailedCount(0);
+ }
+ } else {
+ // Evey
+ iMonitorID.setLastMonitored(new Timestamp((new Date()).getTime()));
+ // if the job is complete we remove it from the Map, if any of these maps
+ // get empty this userMonitorData will get delete from the queue
+ }
+ }
+
+ cleanup(take);
+
+
+ } else {
+ logger.debug("Qstat Monitor doesn't handle non-gsissh hosts , host {}", iHostMonitorData.
+ getComputeResourceDescription().getHostName());
+ }
+ }
+ // We have finished all the HostMonitorData object in userMonitorData, now we need to put it back
+ // now the userMonitorData goes back to the tail of the queue
+ // during individual monitorID removal we remove the HostMonitorData object if it become empty
+ // so if all the jobs are finished for all the hostMOnitorId objects in userMonitorData object
+ // we should remove it from the queue so here we do not put it back.
+ for (ListIterator<HostMonitorData> iterator1 = take.getHostMonitorData().listIterator(); iterator1.hasNext(); ) {
+ HostMonitorData iHostMonitorID = iterator1.next();
+ if (iHostMonitorID.getMonitorIDs().size() == 0) {
+ iterator1.remove();
+ logger.debug("Removed host {} from monitoring queue", iHostMonitorID.getComputeResourceDescription().getHostName());
+ }
+ }
+ if(take.getHostMonitorData().size()!=0) {
+ queue.put(take);
+ }
+ } catch (InterruptedException e) {
+ if (!this.queue.contains(take)) {
+ try {
+ this.queue.put(take);
+ } catch (InterruptedException e1) {
+ e1.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+ }
+ logger.error("Error handling the job with Job ID:" + currentMonitorID.getJobID());
+ throw new AiravataMonitorException(e);
+ } catch (SSHApiException e) {
+ logger.error(e.getMessage());
+ if (e.getMessage().contains("Unknown Job Id Error")) {
+ // in this case job is finished or may be the given job ID is wrong
+ jobStatus.setState(JobState.UNKNOWN);
+ JobIdentifier jobIdentifier = new JobIdentifier("UNKNOWN", "UNKNOWN", "UNKNOWN", "UNKNOWN", "UNKNOWN");
+ if (currentMonitorID != null){
+ jobIdentifier.setExperimentId(currentMonitorID.getExperimentID());
+ jobIdentifier.setTaskId(currentMonitorID.getTaskID());
+ jobIdentifier.setWorkflowNodeId(currentMonitorID.getWorkflowNodeID());
+ jobIdentifier.setJobId(currentMonitorID.getJobID());
+ jobIdentifier.setGatewayId(currentMonitorID.getJobExecutionContext().getGatewayID());
+ }
+ jobStatus.setJobIdentity(jobIdentifier);
+ publisher.publish(jobStatus);
+ } else if (e.getMessage().contains("illegally formed job identifier")) {
+ logger.error("Wrong job ID is given so dropping the job from monitoring system");
+ } else if (!this.queue.contains(take)) {
+ try {
+ queue.put(take);
+ } catch (InterruptedException e1) {
+ e1.printStackTrace();
+ }
+ }
+ throw new AiravataMonitorException("Error retrieving the job status", e);
+ } catch (Exception e) {
+ try {
+ queue.put(take);
+ } catch (InterruptedException e1) {
+ e1.printStackTrace();
+ }
+ throw new AiravataMonitorException("Error retrieving the job status", e);
+ }
+ return true;
+ }
+
+ private void sendNotification(MonitorID iMonitorID) {
+ JobStatusChangeRequestEvent jobStatus = new JobStatusChangeRequestEvent();
+ JobIdentifier jobIdentity = new JobIdentifier(iMonitorID.getJobID(),
+ iMonitorID.getTaskID(),
+ iMonitorID.getWorkflowNodeID(),
+ iMonitorID.getExperimentID(),
+ iMonitorID.getJobExecutionContext().getGatewayID());
+ jobStatus.setJobIdentity(jobIdentity);
+ jobStatus.setState(iMonitorID.getStatus());
+ // we have this JobStatus class to handle amqp monitoring
+ logger.debugId(jobStatus.getJobIdentity().getJobId(), "Published job status change request, " +
+ "experiment {} , task {}", jobStatus.getJobIdentity().getExperimentId(),
+ jobStatus.getJobIdentity().getTaskId());
+
+ publisher.publish(jobStatus);
+ }
+
+ /**
+ * This is the method to stop the polling process
+ *
+ * @return if the stopping process is successful return true else false
+ */
+ public boolean stopPulling() {
+ this.startPulling = false;
+ return true;
+ }
+
+ public MonitorPublisher getPublisher() {
+ return publisher;
+ }
+
+ public void setPublisher(MonitorPublisher publisher) {
+ this.publisher = publisher;
+ }
+
+ public BlockingQueue<UserMonitorData> getQueue() {
+ return queue;
+ }
+
+ public void setQueue(BlockingQueue<UserMonitorData> queue) {
+ this.queue = queue;
+ }
+
+ public boolean authenticate() {
+ return false; //To change body of implemented methods use File | Settings | File Templates.
+ }
+
+ public Map<String, ResourceConnection> getConnections() {
+ return connections;
+ }
+
+ public boolean isStartPulling() {
+ return startPulling;
+ }
+
+ public void setConnections(Map<String, ResourceConnection> connections) {
+ this.connections = connections;
+ }
+
+ public void setStartPulling(boolean startPulling) {
+ this.startPulling = startPulling;
+ }
+
+ public GFac getGfac() {
+ return gfac;
+ }
+
+ public void setGfac(GFac gfac) {
+ this.gfac = gfac;
+ }
+
+ public AuthenticationInfo getAuthenticationInfo() {
+ return authenticationInfo;
+ }
+
+ public void setAuthenticationInfo(AuthenticationInfo authenticationInfo) {
+ this.authenticationInfo = authenticationInfo;
+ }
+
+ public LinkedBlockingQueue<String> getCancelJobList() {
+ return cancelJobList;
+ }
+
+ public void setCancelJobList(LinkedBlockingQueue<String> cancelJobList) {
+ this.cancelJobList = cancelJobList;
+ }
+
+
+ private void cleanup(UserMonitorData userMonitorData){
+ for(MonitorID iMonitorId:removeList){
+ try {
+ CommonUtils.removeMonitorFromQueue(userMonitorData, iMonitorId);
+ } catch (AiravataMonitorException e) {
+ logger.error(e.getMessage(), e);
+ logger.error("Error deleting the monitor data: " + iMonitorId.getJobID());
+ }
+ }
+ removeList.clear();
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java
new file mode 100644
index 0000000..f718535
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java
@@ -0,0 +1,154 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.impl.pull.qstat;
+
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.SecurityContext;
+import org.apache.airavata.gfac.core.monitor.MonitorID;
+import org.apache.airavata.gfac.gsissh.security.GSISecurityContext;
+import org.apache.airavata.gfac.monitor.HostMonitorData;
+import org.apache.airavata.gfac.ssh.security.SSHSecurityContext;
+import org.apache.airavata.gsi.ssh.api.SSHApiException;
+import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
+import org.apache.airavata.gsi.ssh.impl.JobStatus;
+import org.apache.airavata.gsi.ssh.impl.PBSCluster;
+import org.apache.airavata.model.workspace.experiment.JobState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+
+public class ResourceConnection {
+ private static final Logger log = LoggerFactory.getLogger(ResourceConnection.class);
+
+ private PBSCluster cluster;
+
+ private AuthenticationInfo authenticationInfo;
+
+
+ public ResourceConnection(HostMonitorData hostMonitorData,AuthenticationInfo authInfo) throws SSHApiException {
+ MonitorID monitorID = hostMonitorData.getMonitorIDs().get(0);
+ try {
+ SecurityContext securityContext = monitorID.getJobExecutionContext().getSecurityContext(monitorID.getComputeResourceDescription().getHostName());
+ if(securityContext != null) {
+ if (securityContext instanceof GSISecurityContext) {
+ GSISecurityContext gsiSecurityContext = (GSISecurityContext) securityContext;
+ cluster = (PBSCluster) gsiSecurityContext.getPbsCluster();
+ } else if (securityContext instanceof SSHSecurityContext) {
+ SSHSecurityContext sshSecurityContext = (SSHSecurityContext)
+ securityContext;
+ cluster = (PBSCluster) sshSecurityContext.getPbsCluster();
+ }
+ }
+ // we just use cluster configuration from the incoming request and construct a new cluster because for monitoring
+ // we are using our own credentials and not using one users account to do everything.
+ authenticationInfo = authInfo;
+ } catch (GFacException e) {
+ log.error("Error reading data from job ExecutionContext");
+ }
+ }
+
+ public ResourceConnection(HostMonitorData hostMonitorData) throws SSHApiException {
+ MonitorID monitorID = hostMonitorData.getMonitorIDs().get(0);
+ try {
+ GSISecurityContext securityContext = (GSISecurityContext)
+ monitorID.getJobExecutionContext().getSecurityContext(monitorID.getComputeResourceDescription().getHostName());
+ cluster = (PBSCluster) securityContext.getPbsCluster();
+
+ // we just use cluster configuration from the incoming request and construct a new cluster because for monitoring
+ // we are using our own credentials and not using one users account to do everything.
+ cluster = new PBSCluster(cluster.getServerInfo(), authenticationInfo, cluster.getJobManagerConfiguration());
+ } catch (GFacException e) {
+ log.error("Error reading data from job ExecutionContext");
+ }
+ }
+
+ public JobState getJobStatus(MonitorID monitorID) throws SSHApiException {
+ String jobID = monitorID.getJobID();
+ //todo so currently we execute the qstat for each job but we can use user based monitoring
+ //todo or we should concatenate all the commands and execute them in one go and parseSingleJob the response
+ return getStatusFromString(cluster.getJobStatus(jobID).toString());
+ }
+
+ public Map<String, JobState> getJobStatuses(List<MonitorID> monitorIDs) throws SSHApiException {
+ Map<String, JobStatus> treeMap = new TreeMap<String, JobStatus>();
+ Map<String, JobState> treeMap1 = new TreeMap<String, JobState>();
+ // creating a sorted map with all the jobIds and with the predefined
+ // status as UNKNOWN
+ for (MonitorID monitorID : monitorIDs) {
+ treeMap.put(monitorID.getJobID()+","+monitorID.getJobName(), JobStatus.U);
+ }
+ String userName = cluster.getServerInfo().getUserName();
+ //todo so currently we execute the qstat for each job but we can use user based monitoring
+ //todo or we should concatenate all the commands and execute them in one go and parseSingleJob the response
+ //
+ cluster.getJobStatuses(userName, treeMap);
+ for (String key : treeMap.keySet()) {
+ treeMap1.put(key, getStatusFromString(treeMap.get(key).toString()));
+ }
+ return treeMap1;
+ }
+
+ private JobState getStatusFromString(String status) {
+ log.info("parsing the job status returned : " + status);
+ if (status != null) {
+ if ("C".equals(status) || "CD".equals(status) || "E".equals(status) || "CG".equals(status) || "DONE".equals(status)) {
+ return JobState.COMPLETE;
+ } else if ("H".equals(status) || "h".equals(status)) {
+ return JobState.HELD;
+ } else if ("Q".equals(status) || "qw".equals(status) || "PEND".equals(status)) {
+ return JobState.QUEUED;
+ } else if ("R".equals(status) || "CF".equals(status) || "r".equals(status) || "RUN".equals(status)) {
+ return JobState.ACTIVE;
+ } else if ("T".equals(status)) {
+ return JobState.HELD;
+ } else if ("W".equals(status) || "PD".equals(status)) {
+ return JobState.QUEUED;
+ } else if ("S".equals(status) || "PSUSP".equals(status) || "USUSP".equals(status) || "SSUSP".equals(status)) {
+ return JobState.SUSPENDED;
+ } else if ("CA".equals(status)) {
+ return JobState.CANCELED;
+ } else if ("F".equals(status) || "NF".equals(status) || "TO".equals(status) || "EXIT".equals(status)) {
+ return JobState.FAILED;
+ } else if ("PR".equals(status) || "Er".equals(status)) {
+ return JobState.FAILED;
+ } else if ("U".equals(status) || ("UNKWN".equals(status))) {
+ return JobState.UNKNOWN;
+ }
+ }
+ return JobState.UNKNOWN;
+ }
+
+ public PBSCluster getCluster() {
+ return cluster;
+ }
+
+ public void setCluster(PBSCluster cluster) {
+ this.cluster = cluster;
+ }
+
+ public boolean isConnected(){
+ return this.cluster.getSession().isConnected();
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/AMQPMonitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/AMQPMonitor.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/AMQPMonitor.java
new file mode 100644
index 0000000..de8cd8c
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/AMQPMonitor.java
@@ -0,0 +1,280 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.impl.push.amqp;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+
+import org.apache.airavata.common.utils.MonitorPublisher;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.gfac.core.monitor.MonitorID;
+import org.apache.airavata.gfac.monitor.core.PushMonitor;
+import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
+import org.apache.airavata.gfac.monitor.util.AMQPConnectionUtil;
+import org.apache.airavata.gfac.monitor.util.CommonUtils;
+import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
+import org.apache.airavata.model.messaging.event.JobIdentifier;
+import org.apache.airavata.model.messaging.event.JobStatusChangeEvent;
+import org.apache.airavata.model.workspace.experiment.JobState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.eventbus.EventBus;
+import com.google.common.eventbus.Subscribe;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+
+/**
+ * This is the implementation for AMQP based finishQueue, this uses
+ * rabbitmq client to recieve AMQP based monitoring data from
+ * mostly excede resources.
+ */
+public class AMQPMonitor extends PushMonitor {
+ private final static Logger logger = LoggerFactory.getLogger(AMQPMonitor.class);
+
+
+ /* this will keep all the channels available in the system, we do not create
+ channels for all the jobs submitted, but we create channels for each user for each
+ host.
+ */
+ private Map<String, Channel> availableChannels;
+
+ private MonitorPublisher publisher;
+
+ private MonitorPublisher localPublisher;
+
+ private BlockingQueue<MonitorID> runningQueue;
+
+ private BlockingQueue<MonitorID> finishQueue;
+
+ private String connectionName;
+
+ private String proxyPath;
+
+ private List<String> amqpHosts;
+
+ private boolean startRegister;
+
+ public AMQPMonitor(){
+
+ }
+ public AMQPMonitor(MonitorPublisher publisher, BlockingQueue<MonitorID> runningQueue,
+ BlockingQueue<MonitorID> finishQueue,
+ String proxyPath,String connectionName,List<String> hosts) {
+ this.publisher = publisher;
+ this.runningQueue = runningQueue; // these will be initialized by the MonitorManager
+ this.finishQueue = finishQueue; // these will be initialized by the MonitorManager
+ this.availableChannels = new HashMap<String, Channel>();
+ this.connectionName = connectionName;
+ this.proxyPath = proxyPath;
+ this.amqpHosts = hosts;
+ this.localPublisher = new MonitorPublisher(new EventBus());
+ this.localPublisher.registerListener(this);
+ }
+
+ public void initialize(String proxyPath, String connectionName, List<String> hosts) {
+ this.availableChannels = new HashMap<String, Channel>();
+ this.connectionName = connectionName;
+ this.proxyPath = proxyPath;
+ this.amqpHosts = hosts;
+ this.localPublisher = new MonitorPublisher(new EventBus());
+ this.localPublisher.registerListener(this);
+ }
+
+ @Override
+ public boolean registerListener(MonitorID monitorID) throws AiravataMonitorException {
+ // we subscribe to read user-host based subscription
+ ComputeResourceDescription computeResourceDescription = monitorID.getComputeResourceDescription();
+ if (computeResourceDescription.isSetIpAddresses() && computeResourceDescription.getIpAddresses().size() > 0) {
+ // we get first ip address for the moment
+ String hostAddress = computeResourceDescription.getIpAddresses().get(0);
+ // in amqp case there are no multiple jobs per each host, because once a job is put in to the queue it
+ // will be picked by the Monitor, so jobs will not stay in this queueu but jobs will stay in finishQueue
+ String channelID = CommonUtils.getChannelID(monitorID);
+ if (availableChannels.get(channelID) == null) {
+ try {
+ //todo need to fix this rather getting it from a file
+ Connection connection = AMQPConnectionUtil.connect(amqpHosts, connectionName, proxyPath);
+ Channel channel = null;
+ channel = connection.createChannel();
+ availableChannels.put(channelID, channel);
+ String queueName = channel.queueDeclare().getQueue();
+
+ BasicConsumer consumer = new
+ BasicConsumer(new JSONMessageParser(), localPublisher); // here we use local publisher
+ channel.basicConsume(queueName, true, consumer);
+ String filterString = CommonUtils.getRoutingKey(monitorID.getUserName(), hostAddress);
+ // here we queuebind to a particular user in a particular machine
+ channel.queueBind(queueName, "glue2.computing_activity", filterString);
+ logger.info("Using filtering string to monitor: " + filterString);
+ } catch (IOException e) {
+ logger.error("Error creating the connection to finishQueue the job:" + monitorID.getUserName());
+ }
+ }
+ } else {
+ throw new AiravataMonitorException("Couldn't register monitor for jobId :" + monitorID.getJobID() +
+ " , ComputeResourceDescription " + computeResourceDescription.getHostName() + " doesn't has an " +
+ "IpAddress with it");
+ }
+ return true;
+ }
+
+ public void run() {
+ // before going to the while true mode we start unregister thread
+ startRegister = true; // this will be unset by someone else
+ while (startRegister || !ServerSettings.isStopAllThreads()) {
+ try {
+ MonitorID take = runningQueue.take();
+ this.registerListener(take);
+ } catch (AiravataMonitorException e) { // catch any exceptino inside the loop
+ logger.error(e.getMessage(), e);
+ } catch (InterruptedException e) {
+ logger.error(e.getMessage(), e);
+ } catch (Exception e){
+ logger.error(e.getMessage(), e);
+ }
+ }
+ Set<String> strings = availableChannels.keySet();
+ for(String key:strings) {
+ Channel channel = availableChannels.get(key);
+ try {
+ channel.close();
+ } catch (IOException e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+ }
+
+ @Subscribe
+ public boolean unRegisterListener(MonitorID monitorID) throws AiravataMonitorException {
+ Iterator<MonitorID> iterator = finishQueue.iterator();
+ MonitorID next = null;
+ while(iterator.hasNext()){
+ next = iterator.next();
+ if(next.getJobID().endsWith(monitorID.getJobID())){
+ break;
+ }
+ }
+ if(next == null) {
+ logger.error("Job has removed from the queue, old obsolete message recieved");
+ return false;
+ }
+ String channelID = CommonUtils.getChannelID(next);
+ if (JobState.FAILED.equals(monitorID.getStatus()) || JobState.COMPLETE.equals(monitorID.getStatus())) {
+ finishQueue.remove(next);
+
+ // if this is the last job in the queue at this point with the same username and same host we
+ // close the channel and close the connection and remove it from availableChannels
+ if (CommonUtils.isTheLastJobInQueue(finishQueue, next)) {
+ logger.info("There are no jobs to monitor for common ChannelID:" + channelID + " , so we unsubscribe it" +
+ ", incase new job created we do subscribe again");
+ Channel channel = availableChannels.get(channelID);
+ if (channel == null) {
+ logger.error("Already Unregistered the listener");
+ throw new AiravataMonitorException("Already Unregistered the listener");
+ } else {
+ try {
+ channel.queueUnbind(channel.queueDeclare().getQueue(), "glue2.computing_activity", CommonUtils.getRoutingKey(next));
+ channel.close();
+ channel.getConnection().close();
+ availableChannels.remove(channelID);
+ } catch (IOException e) {
+ logger.error("Error unregistering the listener");
+ throw new AiravataMonitorException("Error unregistering the listener");
+ }
+ }
+ }
+ }
+ next.setStatus(monitorID.getStatus());
+ JobIdentifier jobIdentity = new JobIdentifier(next.getJobID(),
+ next.getTaskID(),
+ next.getWorkflowNodeID(),
+ next.getExperimentID(),
+ next.getJobExecutionContext().getGatewayID());
+ publisher.publish(new JobStatusChangeEvent(next.getStatus(),jobIdentity));
+ return true;
+ }
+ @Override
+ public boolean stopRegister() throws AiravataMonitorException {
+ return false; //To change body of implemented methods use File | Settings | File Templates.
+ }
+
+ public Map<String, Channel> getAvailableChannels() {
+ return availableChannels;
+ }
+
+ public void setAvailableChannels(Map<String, Channel> availableChannels) {
+ this.availableChannels = availableChannels;
+ }
+
+ public MonitorPublisher getPublisher() {
+ return publisher;
+ }
+
+ public void setPublisher(MonitorPublisher publisher) {
+ this.publisher = publisher;
+ }
+
+ public BlockingQueue<MonitorID> getRunningQueue() {
+ return runningQueue;
+ }
+
+ public void setRunningQueue(BlockingQueue<MonitorID> runningQueue) {
+ this.runningQueue = runningQueue;
+ }
+
+ public BlockingQueue<MonitorID> getFinishQueue() {
+ return finishQueue;
+ }
+
+ public void setFinishQueue(BlockingQueue<MonitorID> finishQueue) {
+ this.finishQueue = finishQueue;
+ }
+
+ public String getProxyPath() {
+ return proxyPath;
+ }
+
+ public void setProxyPath(String proxyPath) {
+ this.proxyPath = proxyPath;
+ }
+
+ public List<String> getAmqpHosts() {
+ return amqpHosts;
+ }
+
+ public void setAmqpHosts(List<String> amqpHosts) {
+ this.amqpHosts = amqpHosts;
+ }
+
+ public boolean isStartRegister() {
+ return startRegister;
+ }
+
+ public void setStartRegister(boolean startRegister) {
+ this.startRegister = startRegister;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/BasicConsumer.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/BasicConsumer.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/BasicConsumer.java
new file mode 100644
index 0000000..bd5c625
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/BasicConsumer.java
@@ -0,0 +1,87 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.impl.push.amqp;
+
+import org.apache.airavata.common.utils.MonitorPublisher;
+import org.apache.airavata.gfac.core.monitor.MonitorID;
+import org.apache.airavata.gfac.monitor.core.MessageParser;
+import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Consumer;
+import com.rabbitmq.client.Envelope;
+import com.rabbitmq.client.ShutdownSignalException;
+
+public class BasicConsumer implements Consumer {
+ private final static Logger logger = LoggerFactory.getLogger(AMQPMonitor.class);
+
+ private MessageParser parser;
+
+ private MonitorPublisher publisher;
+
+ public BasicConsumer(MessageParser parser, MonitorPublisher publisher) {
+ this.parser = parser;
+ this.publisher = publisher;
+ }
+
+ public void handleCancel(String consumerTag) {
+ }
+
+ public void handleCancelOk(String consumerTag) {
+ }
+
+ public void handleConsumeOk(String consumerTag) {
+ }
+
+ public void handleDelivery(String consumerTag,
+ Envelope envelope,
+ AMQP.BasicProperties properties,
+ byte[] body) {
+
+ logger.debug("job update for: " + envelope.getRoutingKey());
+ String message = new String(body);
+ message = message.replaceAll("(?m)^", " ");
+ // Here we parse the message and get the job status and push it
+ // to the Event bus, this will be picked by
+// AiravataJobStatusUpdator and store in to registry
+
+ logger.debug("************************************************************");
+ logger.debug("AMQP Message recieved \n" + message);
+ logger.debug("************************************************************");
+ try {
+ String jobID = envelope.getRoutingKey().split("\\.")[0];
+ MonitorID monitorID = new MonitorID(null, jobID, null, null, null, null,null);
+ monitorID.setStatus(parser.parseMessage(message));
+ publisher.publish(monitorID);
+ } catch (AiravataMonitorException e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+
+ public void handleRecoverOk(String consumerTag) {
+ }
+
+ public void handleShutdownSignal(String consumerTag, ShutdownSignalException sig) {
+ }
+
+}
[5/7] airavata git commit: Add new gfac email monitor module to gfac
monitor module and Intorduced gfac-hpc-monitor module to keep all hoc-monitor
code
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/JSONMessageParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/JSONMessageParser.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/JSONMessageParser.java
new file mode 100644
index 0000000..72c77d5
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/JSONMessageParser.java
@@ -0,0 +1,78 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.impl.push.amqp;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.airavata.ComputingActivity;
+import org.apache.airavata.gfac.monitor.core.MessageParser;
+import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
+import org.apache.airavata.model.workspace.experiment.JobState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+
+public class JSONMessageParser implements MessageParser {
+ private final static Logger logger = LoggerFactory.getLogger(JSONMessageParser.class);
+
+ public JobState parseMessage(String message)throws AiravataMonitorException {
+ /*todo write a json message parser here*/
+ logger.debug(message);
+ ObjectMapper objectMapper = new ObjectMapper();
+ try {
+ ComputingActivity computingActivity = objectMapper.readValue(message.getBytes(), ComputingActivity.class);
+ logger.info(computingActivity.getIDFromEndpoint());
+ List<String> stateList = computingActivity.getState();
+ JobState jobState = null;
+ for (String aState : stateList) {
+ jobState = getStatusFromString(aState);
+ }
+ // we get the last value of the state array
+ return jobState;
+ } catch (IOException e) {
+ throw new AiravataMonitorException(e);
+ }
+ }
+
+private JobState getStatusFromString(String status) {
+ logger.info("parsing the job status returned : " + status);
+ if(status != null){
+ if("ipf:finished".equals(status)){
+ return JobState.COMPLETE;
+ }else if("ipf:pending".equals(status)|| "ipf:starting".equals(status)){
+ return JobState.QUEUED;
+ }else if("ipf:running".equals(status) || "ipf:finishing".equals(status)){
+ return JobState.ACTIVE;
+ }else if ("ipf:held".equals(status) || "ipf:teminating".equals(status) || "ipf:teminated".equals(status)) {
+ return JobState.HELD;
+ } else if ("ipf:suspending".equals(status)) {
+ return JobState.SUSPENDED;
+ }else if ("ipf:failed".equals(status)) {
+ return JobState.FAILED;
+ }else if ("ipf:unknown".equals(status)){
+ return JobState.UNKNOWN;
+ }
+ }
+ return JobState.UNKNOWN;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/SimpleJobFinishConsumer.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/SimpleJobFinishConsumer.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/SimpleJobFinishConsumer.java
new file mode 100644
index 0000000..c4275f1
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/SimpleJobFinishConsumer.java
@@ -0,0 +1,86 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.impl.push.amqp;
+
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import org.apache.airavata.common.utils.Constants;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+
+public class SimpleJobFinishConsumer {
+ private final static Logger logger = LoggerFactory.getLogger(SimpleJobFinishConsumer.class);
+
+ private List<String> completedJobsFromPush;
+
+ public SimpleJobFinishConsumer(List<String> completedJobsFromPush) {
+ this.completedJobsFromPush = completedJobsFromPush;
+ }
+
+ public void listen() {
+ try {
+ String queueName = ServerSettings.getSetting(Constants.GFAC_SERVER_PORT, "8950");
+ String uri = "amqp://localhost";
+
+ ConnectionFactory connFactory = new ConnectionFactory();
+ connFactory.setUri(uri);
+ Connection conn = connFactory.newConnection();
+ logger.info("--------Created the connection to Rabbitmq server successfully-------");
+
+ final Channel ch = conn.createChannel();
+
+ logger.info("--------Created the channel with Rabbitmq server successfully-------");
+
+ ch.queueDeclare(queueName, false, false, false, null);
+
+ logger.info("--------Declare the queue " + queueName + " in Rabbitmq server successfully-------");
+
+ final QueueingConsumer consumer = new QueueingConsumer(ch);
+ ch.basicConsume(queueName, consumer);
+ (new Thread() {
+ public void run() {
+ try {
+ while (true) {
+ QueueingConsumer.Delivery delivery = consumer.nextDelivery();
+ String message = new String(delivery.getBody());
+ logger.info("---------------- Job Finish message received:" + message + " --------------");
+ synchronized (completedJobsFromPush) {
+ completedJobsFromPush.add(message);
+ }
+ ch.basicAck(delivery.getEnvelope().getDeliveryTag(), false);
+ }
+ } catch (Exception ex) {
+ logger.error("--------Cannot connect to a RabbitMQ Server--------" , ex);
+ }
+ }
+
+ }).start();
+ } catch (Exception ex) {
+ logger.error("Cannot connect to a RabbitMQ Server: " , ex);
+ logger.info("------------- Push monitoring for HPC jobs is disabled -------------");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/UnRegisterWorker.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/UnRegisterWorker.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/UnRegisterWorker.java
new file mode 100644
index 0000000..a701326
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/UnRegisterWorker.java
@@ -0,0 +1,67 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.impl.push.amqp;
+
+import com.google.common.eventbus.Subscribe;
+import com.rabbitmq.client.Channel;
+import org.apache.airavata.gfac.core.monitor.MonitorID;
+import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
+import org.apache.airavata.gfac.monitor.util.CommonUtils;
+import org.apache.airavata.model.messaging.event.JobStatusChangeEvent;
+import org.apache.airavata.model.workspace.experiment.JobState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class UnRegisterWorker{
+ private final static Logger logger = LoggerFactory.getLogger(UnRegisterWorker.class);
+ private Map<String, Channel> availableChannels;
+
+ public UnRegisterWorker(Map<String, Channel> channels) {
+ this.availableChannels = channels;
+ }
+
+ @Subscribe
+ private boolean unRegisterListener(JobStatusChangeEvent jobStatus, MonitorID monitorID) throws AiravataMonitorException {
+ String channelID = CommonUtils.getChannelID(monitorID);
+ if (JobState.FAILED.equals(jobStatus.getState()) || JobState.COMPLETE.equals(jobStatus.getState())){
+ Channel channel = availableChannels.get(channelID);
+ if (channel == null) {
+ logger.error("Already Unregistered the listener");
+ throw new AiravataMonitorException("Already Unregistered the listener");
+ } else {
+ try {
+ channel.queueUnbind(channel.queueDeclare().getQueue(), "glue2.computing_activity", CommonUtils.getRoutingKey(monitorID));
+ channel.close();
+ channel.getConnection().close();
+ availableChannels.remove(channelID);
+ } catch (IOException e) {
+ logger.error("Error unregistering the listener");
+ throw new AiravataMonitorException("Error unregistering the listener");
+ }
+ }
+ }
+ return true;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/AMQPConnectionUtil.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/AMQPConnectionUtil.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/AMQPConnectionUtil.java
new file mode 100644
index 0000000..6a4ed3b
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/AMQPConnectionUtil.java
@@ -0,0 +1,80 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.util;
+
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.DefaultSaslConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManagerFactory;
+import java.security.KeyStore;
+import java.util.Collections;
+import java.util.List;
+
+public class AMQPConnectionUtil {
+ private final static Logger logger = LoggerFactory.getLogger(AMQPConnectionUtil.class);
+ public static Connection connect(List<String>hosts,String vhost, String proxyFile) {
+ Collections.shuffle(hosts);
+ for (String host : hosts) {
+ Connection connection = connect(host, vhost, proxyFile);
+ if (host != null) {
+ System.out.println("connected to " + host);
+ return connection;
+ }
+ }
+ return null;
+ }
+
+ public static Connection connect(String host, String vhost, String proxyFile) {
+ Connection connection;
+ try {
+ String keyPassPhrase = "test123";
+ KeyStore ks = X509Helper.keyStoreFromPEM(proxyFile, keyPassPhrase);
+ KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509");
+ kmf.init(ks, keyPassPhrase.toCharArray());
+
+ KeyStore tks = X509Helper.trustKeyStoreFromCertDir();
+ TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509");
+ tmf.init(tks);
+
+ SSLContext c = SSLContext.getInstance("SSLv3");
+ c.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null);
+
+ ConnectionFactory factory = new ConnectionFactory();
+ factory.setHost(host);
+ factory.setPort(5671);
+ factory.useSslProtocol(c);
+ factory.setVirtualHost(vhost);
+ factory.setSaslConfig(DefaultSaslConfig.EXTERNAL);
+
+ connection = factory.newConnection();
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ return null;
+ }
+ return connection;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/CommonUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/CommonUtils.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/CommonUtils.java
new file mode 100644
index 0000000..15b7241
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/CommonUtils.java
@@ -0,0 +1,298 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.util;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.logger.AiravataLogger;
+import org.apache.airavata.common.logger.AiravataLoggerFactory;
+import org.apache.airavata.common.utils.AiravataZKUtils;
+import org.apache.airavata.common.utils.Constants;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.core.context.JobExecutionContext;
+import org.apache.airavata.gfac.core.handler.GFacHandler;
+import org.apache.airavata.gfac.core.handler.GFacHandlerConfig;
+import org.apache.airavata.gfac.core.monitor.MonitorID;
+import org.apache.airavata.gfac.monitor.HostMonitorData;
+import org.apache.airavata.gfac.monitor.UserMonitorData;
+import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
+import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
+import org.apache.zookeeper.*;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CountDownLatch;
+
+public class CommonUtils {
+ private final static AiravataLogger logger = AiravataLoggerFactory.getLogger(CommonUtils.class);
+
+ public static String getChannelID(MonitorID monitorID) {
+ return monitorID.getUserName() + "-" + monitorID.getComputeResourceDescription().getHostName();
+ }
+
+ public static String getRoutingKey(MonitorID monitorID) {
+ return "*." + monitorID.getUserName() + "." + monitorID.getComputeResourceDescription().getIpAddresses().get(0);
+ }
+
+ public static String getChannelID(String userName,String hostAddress) {
+ return userName + "-" + hostAddress;
+ }
+
+ public static String getRoutingKey(String userName,String hostAddress) {
+ return "*." + userName + "." + hostAddress;
+ }
+
+ public static void addMonitortoQueue(BlockingQueue<UserMonitorData> queue, MonitorID monitorID, JobExecutionContext jobExecutionContext) throws AiravataMonitorException {
+ synchronized (queue) {
+ Iterator<UserMonitorData> iterator = queue.iterator();
+ while (iterator.hasNext()) {
+ UserMonitorData next = iterator.next();
+ if (next.getUserName().equals(monitorID.getUserName())) {
+ // then this is the right place to update
+ List<HostMonitorData> monitorIDs = next.getHostMonitorData();
+ for (HostMonitorData host : monitorIDs) {
+ if (isEqual(host.getComputeResourceDescription(), monitorID.getComputeResourceDescription())) {
+ // ok we found right place to add this monitorID
+ host.addMonitorIDForHost(monitorID);
+ logger.debugId(monitorID.getJobID(), "Added new job to the monitoring queue, experiment {}," +
+ " task {}", monitorID.getExperimentID(), monitorID.getTaskID());
+ return;
+ }
+ }
+ // there is a userMonitor object for this user name but no Hosts for this host
+ // so we have to create new Hosts
+ HostMonitorData hostMonitorData = new HostMonitorData(jobExecutionContext);
+ hostMonitorData.addMonitorIDForHost(monitorID);
+ next.addHostMonitorData(hostMonitorData);
+ logger.debugId(monitorID.getJobID(), "Added new job to the monitoring queue, experiment {}," +
+ " task {}", monitorID.getExperimentID(), monitorID.getTaskID());
+ return;
+ }
+ }
+ HostMonitorData hostMonitorData = new HostMonitorData(jobExecutionContext);
+ hostMonitorData.addMonitorIDForHost(monitorID);
+
+ UserMonitorData userMonitorData = new UserMonitorData(monitorID.getUserName());
+ userMonitorData.addHostMonitorData(hostMonitorData);
+ try {
+ queue.put(userMonitorData);
+ logger.debugId(monitorID.getJobID(), "Added new job to the monitoring queue, experiment {}," +
+ " task {}", monitorID.getExperimentID(), monitorID.getTaskID());
+ } catch (InterruptedException e) {
+ throw new AiravataMonitorException(e);
+ }
+ }
+ }
+
+ private static boolean isEqual(ComputeResourceDescription comRes_1, ComputeResourceDescription comRes_2) {
+ return comRes_1.getComputeResourceId().equals(comRes_2.getComputeResourceId()) &&
+ comRes_1.getHostName().equals(comRes_2.getHostName());
+ }
+
+ public static boolean isTheLastJobInQueue(BlockingQueue<MonitorID> queue,MonitorID monitorID){
+ Iterator<MonitorID> iterator = queue.iterator();
+ while(iterator.hasNext()){
+ MonitorID next = iterator.next();
+ if (monitorID.getUserName().equals(next.getUserName()) &&
+ CommonUtils.isEqual(monitorID.getComputeResourceDescription(), next.getComputeResourceDescription())) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * This method doesn't have to be synchronized because it will be invoked by HPCPullMonitor which already synchronized
+ * @param monitorID
+ * @throws AiravataMonitorException
+ */
+ public static void removeMonitorFromQueue(UserMonitorData userMonitorData, MonitorID monitorID) throws AiravataMonitorException {
+ if (userMonitorData.getUserName().equals(monitorID.getUserName())) {
+ // then this is the right place to update
+ List<HostMonitorData> hostMonitorData = userMonitorData.getHostMonitorData();
+ Iterator<HostMonitorData> iterator1 = hostMonitorData.iterator();
+ while (iterator1.hasNext()) {
+ HostMonitorData iHostMonitorID = iterator1.next();
+ if (isEqual(iHostMonitorID.getComputeResourceDescription(), monitorID.getComputeResourceDescription())) {
+ Iterator<MonitorID> iterator2 = iHostMonitorID.getMonitorIDs().iterator();
+ while (iterator2.hasNext()) {
+ MonitorID iMonitorID = iterator2.next();
+ if (iMonitorID.getJobID().equals(monitorID.getJobID())
+ || iMonitorID.getJobName().equals(monitorID.getJobName())) {
+ // OK we found the object, we cannot do list.remove(object) states of two objects
+ // could be different, thats why we check the jobID
+ iterator2.remove();
+ logger.infoId(monitorID.getJobID(), "Removed the jobId: {} JobName: {} from monitoring last " +
+ "status:{}", monitorID.getJobID(),monitorID.getJobName(), monitorID.getStatus().toString());
+
+ return;
+ }
+ }
+ }
+ }
+ }
+ logger.info("Cannot find the given MonitorID in the queue with userName " +
+ monitorID.getUserName() + " and jobID " + monitorID.getJobID());
+ logger.info("This might not be an error because someone else removed this job from the queue");
+ }
+
+
+ public static void invokeOutFlowHandlers(JobExecutionContext jobExecutionContext) throws GFacException {
+ List<GFacHandlerConfig> handlers = jobExecutionContext.getGFacConfiguration().getOutHandlers();
+
+ for (GFacHandlerConfig handlerClassName : handlers) {
+ Class<? extends GFacHandler> handlerClass;
+ GFacHandler handler;
+ try {
+ handlerClass = Class.forName(handlerClassName.getClassName().trim()).asSubclass(GFacHandler.class);
+ handler = handlerClass.newInstance();
+ handler.initProperties(handlerClassName.getProperties());
+ } catch (ClassNotFoundException e) {
+ logger.error(e.getMessage());
+ throw new GFacException("Cannot load handler class " + handlerClassName, e);
+ } catch (InstantiationException e) {
+ logger.error(e.getMessage());
+ throw new GFacException("Cannot instantiate handler class " + handlerClassName, e);
+ } catch (IllegalAccessException e) {
+ logger.error(e.getMessage());
+ throw new GFacException("Cannot instantiate handler class " + handlerClassName, e);
+ }
+ try {
+ handler.invoke(jobExecutionContext);
+ } catch (Exception e) {
+ // TODO: Better error reporting.
+ throw new GFacException("Error Executing a OutFlow Handler", e);
+ }
+ }
+ }
+
+ /**
+ * Update job count for a given set of paths.
+ * @param zk - zookeeper instance
+ * @param changeCountMap - map of change job count with relevant path
+ * @param isAdd - Should add or reduce existing job count by the given job count.
+ */
+ public static void updateZkWithJobCount(ZooKeeper zk, final Map<String, Integer> changeCountMap, boolean isAdd) {
+ StringBuilder changeZNodePaths = new StringBuilder();
+ try {
+ if (zk == null || !zk.getState().isConnected()) {
+ try {
+ final CountDownLatch countDownLatch = new CountDownLatch(1);
+ zk = new ZooKeeper(AiravataZKUtils.getZKhostPort(), AiravataZKUtils.getZKTimeout(), new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ countDownLatch.countDown();
+ }
+ });
+ countDownLatch.await();
+ } catch (ApplicationSettingsException e) {
+ logger.error("Error while reading zookeeper hostport string");
+ } catch (IOException e) {
+ logger.error("Error while reconnect attempt to zookeeper where zookeeper connection loss state");
+ }
+ }
+
+ for (String path : changeCountMap.keySet()) {
+ if (isAdd) {
+ CommonUtils.checkAndCreateZNode(zk, path);
+ }
+ byte[] byteData = zk.getData(path, null, null);
+ String nodeData;
+ if (byteData == null) {
+ if (isAdd) {
+ zk.setData(path, String.valueOf(changeCountMap.get(path)).getBytes(), -1);
+ } else {
+ // This is not possible, but we handle in case there any data zookeeper communication failure
+ logger.warn("Couldn't reduce job count in " + path + " as it returns null data. Hence reset the job count to 0");
+ zk.setData(path, "0".getBytes(), -1);
+ }
+ } else {
+ nodeData = new String(byteData);
+ if (isAdd) {
+ zk.setData(path, String.valueOf(changeCountMap.get(path) + Integer.parseInt(nodeData)).getBytes(), -1);
+ } else {
+ int previousCount = Integer.parseInt(nodeData);
+ int removeCount = changeCountMap.get(path);
+ if (previousCount >= removeCount) {
+ zk.setData(path, String.valueOf(previousCount - removeCount).getBytes(), -1);
+ } else {
+ // This is not possible, do we need to reset the job count to 0 ?
+ logger.error("Requested remove job count is " + removeCount +
+ " which is higher than the existing job count " + previousCount
+ + " in " + path + " path.");
+ }
+ }
+ }
+ changeZNodePaths.append(path).append(":");
+ }
+
+ // update stat node to trigger orchestrator watchers
+ if (changeCountMap.size() > 0) {
+ changeZNodePaths.deleteCharAt(changeZNodePaths.length() - 1);
+ zk.setData("/" + Constants.STAT, changeZNodePaths.toString().getBytes(), -1);
+ }
+ } catch (KeeperException e) {
+ logger.error("Error while writing job count to zookeeper", e);
+ } catch (InterruptedException e) {
+ logger.error("Error while writing job count to zookeeper", e);
+ }
+
+ }
+
+ /**
+ * Increase job count by one and update the zookeeper
+ * @param monitorID - Job monitorId
+ */
+ public static void increaseZkJobCount(MonitorID monitorID) {
+ Map<String, Integer> addMap = new HashMap<String, Integer>();
+ addMap.put(CommonUtils.getJobCountUpdatePath(monitorID), 1);
+ updateZkWithJobCount(monitorID.getJobExecutionContext().getZk(), addMap, true);
+ }
+
+ /**
+ * Construct and return the path for a given MonitorID , eg: /stat/{username}/{resourceName}/job
+ * @param monitorID - Job monitorId
+ * @return
+ */
+ public static String getJobCountUpdatePath(MonitorID monitorID){
+ return new StringBuilder("/").append(Constants.STAT).append("/").append(monitorID.getUserName())
+ .append("/").append(monitorID.getComputeResourceDescription().getHostName()).append("/").append(Constants.JOB).toString();
+ }
+
+ /**
+ * Check whether znode is exist in given path if not create a new znode
+ * @param zk - zookeeper instance
+ * @param path - path to check znode
+ * @throws KeeperException
+ * @throws InterruptedException
+ */
+ private static void checkAndCreateZNode(ZooKeeper zk , String path) throws KeeperException, InterruptedException {
+ if (zk.exists(path, null) == null) { // if znode doesn't exist
+ if (path.lastIndexOf("/") > 1) { // recursively traverse to parent znode and check parent exist
+ checkAndCreateZNode(zk, (path.substring(0, path.lastIndexOf("/"))));
+ }
+ zk.create(path, null, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);// create a znode
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/X509Helper.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/X509Helper.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/X509Helper.java
new file mode 100644
index 0000000..08c3f67
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/X509Helper.java
@@ -0,0 +1,164 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.util;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
+
+
+import java.io.*;
+import java.security.*;
+import java.security.cert.CertificateException;
+import java.security.cert.CertificateFactory;
+import java.security.cert.CertificateParsingException;
+import java.security.cert.X509Certificate;
+import java.security.spec.InvalidKeySpecException;
+
+public class X509Helper {
+
+ static {
+ // parsing of RSA key fails without this
+ java.security.Security.addProvider(new BouncyCastleProvider());
+ }
+
+
+
+ public static KeyStore keyStoreFromPEM(String proxyFile,
+ String keyPassPhrase) throws IOException,
+ CertificateException,
+ NoSuchAlgorithmException,
+ InvalidKeySpecException,
+ KeyStoreException {
+ return keyStoreFromPEM(proxyFile,proxyFile,keyPassPhrase);
+ }
+
+ public static KeyStore keyStoreFromPEM(String certFile,
+ String keyFile,
+ String keyPassPhrase) throws IOException,
+ CertificateException,
+ NoSuchAlgorithmException,
+ InvalidKeySpecException,
+ KeyStoreException {
+ CertificateFactory cf = CertificateFactory.getInstance("X.509");
+ X509Certificate cert = (X509Certificate)cf.generateCertificate(new FileInputStream(certFile));
+ //System.out.println(cert.toString());
+
+ // this works for proxy files, too, since it skips over the certificate
+ BufferedReader reader = new BufferedReader(new FileReader(keyFile));
+ String line = null;
+ StringBuilder builder = new StringBuilder();
+ boolean inKey = false;
+ while((line=reader.readLine()) != null) {
+ if (line.contains("-----BEGIN RSA PRIVATE KEY-----")) {
+ inKey = true;
+ }
+ if (inKey) {
+ builder.append(line);
+ builder.append(System.getProperty("line.separator"));
+ }
+ if (line.contains("-----END RSA PRIVATE KEY-----")) {
+ inKey = false;
+ }
+ }
+ String privKeyPEM = builder.toString();
+ //System.out.println(privKeyPEM);
+
+ // using BouncyCastle
+// PEMReader pemParser = new PEMReader(new StringReader(privKeyPEM));
+// Object object = pemParser.readObject();
+//
+// PrivateKey privKey = null;
+// if(object instanceof KeyPair){
+// privKey = ((KeyPair)object).getPrivate();
+// }
+ // PEMParser from BouncyCastle is good for reading PEM files, but I didn't want to add that dependency
+ /*
+ // Base64 decode the data
+ byte[] encoded = javax.xml.bind.DatatypeConverter.parseBase64Binary(privKeyPEM);
+
+ // PKCS8 decode the encoded RSA private key
+ java.security.spec.PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(encoded);
+ KeyFactory kf = KeyFactory.getInstance("RSA");
+ PrivateKey privKey = kf.generatePrivate(keySpec);
+ //RSAPrivateKey privKey = (RSAPrivateKey)kf.generatePrivate(keySpec);
+ */
+ //System.out.println(privKey.toString());
+
+// KeyStore keyStore = KeyStore.getInstance("PKCS12");
+// keyStore.load(null,null);
+//
+// KeyStore.PrivateKeyEntry entry =
+// new KeyStore.PrivateKeyEntry(privKey,
+// new java.security.cert.Certificate[] {(java.security.cert.Certificate)cert});
+// KeyStore.PasswordProtection prot = new KeyStore.PasswordProtection(keyPassPhrase.toCharArray());
+// keyStore.setEntry(cert.getSubjectX500Principal().getName(), entry, prot);
+
+// return keyStore;
+ //TODO: Problem with BouncyCastle version used in gsissh
+ throw new CertificateException("Method not implemented");
+
+ }
+
+
+ public static KeyStore trustKeyStoreFromCertDir() throws IOException,
+ KeyStoreException,
+ CertificateException,
+ NoSuchAlgorithmException, ApplicationSettingsException {
+ return trustKeyStoreFromCertDir(ServerSettings.getSetting("trusted.cert.location"));
+ }
+
+ public static KeyStore trustKeyStoreFromCertDir(String certDir) throws IOException,
+ KeyStoreException,
+ CertificateException,
+ NoSuchAlgorithmException {
+ KeyStore ks = KeyStore.getInstance("JKS");
+ ks.load(null,null);
+
+ File dir = new File(certDir);
+ for(File file : dir.listFiles()) {
+ if (!file.isFile()) {
+ continue;
+ }
+ if (!file.getName().endsWith(".0")) {
+ continue;
+ }
+
+ try {
+ //System.out.println("reading file "+file.getName());
+ CertificateFactory cf = CertificateFactory.getInstance("X.509");
+ X509Certificate cert = (X509Certificate) cf.generateCertificate(new FileInputStream(file));
+ //System.out.println(cert.toString());
+
+ KeyStore.TrustedCertificateEntry entry = new KeyStore.TrustedCertificateEntry(cert);
+
+ ks.setEntry(cert.getSubjectX500Principal().getName(), entry, null);
+ } catch (KeyStoreException e) {
+ } catch (CertificateParsingException e) {
+ continue;
+ }
+
+ }
+
+ return ks;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/errors.properties
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/errors.properties b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/errors.properties
new file mode 100644
index 0000000..88c41b8
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/errors.properties
@@ -0,0 +1,197 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Directly copied from jglobus. Not a good way to manager error properties.
+1 = Parameter not supported
+2 = The RSL length is greater than the maximum allowed
+3 = No resources available
+4 = Bad directory specified
+5 = The executable does not exist
+6 = Insufficient funds
+7 = Authentication with the remote server failed
+8 = Job cancelled by user
+9 = Job cancelled by system
+
+10 = Data transfer to the server failed
+11 = The stdin file does not exist
+12 = The connection to the server failed (check host and port)
+13 = The provided RSL 'maxtime' value is invalid (not an integer or must be greater than 0)
+14 = The provided RSL 'count' value is invalid (not an integer or must be greater than 0)
+15 = The job manager received an invalid RSL
+16 = Could not connect to job manager
+17 = The job failed when the job manager attempted to run it
+18 = Paradyn error
+19 = The provided RSL 'jobtype' value is invalid
+
+20 = The provided RSL 'myjob' value is invalid
+21 = The job manager failed to locate an internal script argument file
+22 = The job manager failed to create an internal script argument file
+23 = The job manager detected an invalid job state
+24 = The job manager detected an invalid script response
+25 = The job manager detected an invalid job state
+26 = The provided RSL 'jobtype' value is not supported by this job manager
+27 = Unimplemented
+28 = The job manager failed to create an internal script submission file
+29 = The job manager cannot find the user proxy
+
+30 = The job manager failed to open the user proxy
+31 = The job manager failed to cancel the job as requested
+32 = System memory allocation failed
+33 = The interprocess job communication initialization failed
+34 = The interprocess job communication setup failed
+35 = The provided RSL 'host count' value is invalid
+36 = One of the provided RSL parameters is unsupported
+37 = The provided RSL 'queue' parameter is invalid
+38 = The provided RSL 'project' parameter is invalid
+39 = The provided RSL string includes variables that could not be identified
+
+40 = The provided RSL 'environment' parameter is invalid
+41 = The provided RSL 'dryrun' parameter is invalid
+42 = The provided RSL is invalid (an empty string)
+43 = The job manager failed to stage the executable
+44 = The job manager failed to stage the stdin file
+45 = The requested job manager type is invalid
+46 = The provided RSL 'arguments' parameter is invalid
+47 = The gatekeeper failed to run the job manager
+48 = The provided RSL could not be properly parsed
+49 = There is a version mismatch between GRAM components
+
+50 = The provided RSL 'arguments' parameter is invalid
+51 = The provided RSL 'count' parameter is invalid
+52 = The provided RSL 'directory' parameter is invalid
+53 = The provided RSL 'dryrun' parameter is invalid
+54 = The provided RSL 'environment' parameter is invalid
+55 = The provided RSL 'executable' parameter is invalid
+56 = The provided RSL 'host_count' parameter is invalid
+57 = The provided RSL 'jobtype' parameter is invalid
+58 = The provided RSL 'maxtime' parameter is invalid
+59 = The provided RSL 'myjob' parameter is invalid
+
+60 = The provided RSL 'paradyn' parameter is invalid
+61 = The provided RSL 'project' parameter is invalid
+62 = The provided RSL 'queue' parameter is invalid
+63 = The provided RSL 'stderr' parameter is invalid
+64 = The provided RSL 'stdin' parameter is invalid
+65 = The provided RSL 'stdout' parameter is invalid
+66 = The job manager failed to locate an internal script
+67 = The job manager failed on the system call pipe()
+68 = The job manager failed on the system call fcntl()
+69 = The job manager failed to create the temporary stdout filename
+
+70 = The job manager failed to create the temporary stderr filename
+71 = The job manager failed on the system call fork()
+72 = The executable file permissions do not allow execution
+73 = The job manager failed to open stdout
+74 = The job manager failed to open stderr
+75 = The cache file could not be opened in order to relocate the user proxy
+76 = Cannot access cache files in ~/.globus/.gass_cache, check permissions, quota, and disk space
+77 = The job manager failed to insert the contact in the client contact list
+78 = The contact was not found in the job manager's client contact list
+79 = Connecting to the job manager failed. Possible reasons: job terminated, invalid job contact, network problems, ...
+
+80 = The syntax of the job contact is invalid
+81 = The executable parameter in the RSL is undefined
+82 = The job manager service is misconfigured. condor arch undefined
+83 = The job manager service is misconfigured. condor os undefined
+84 = The provided RSL 'min_memory' parameter is invalid
+85 = The provided RSL 'max_memory' parameter is invalid
+86 = The RSL 'min_memory' value is not zero or greater
+87 = The RSL 'max_memory' value is not zero or greater
+88 = The creation of a HTTP message failed
+89 = Parsing incoming HTTP message failed
+
+90 = The packing of information into a HTTP message failed
+91 = An incoming HTTP message did not contain the expected information
+92 = The job manager does not support the service that the client requested
+93 = The gatekeeper failed to find the requested service
+94 = The jobmanager does not accept any new requests (shutting down)
+95 = The client failed to close the listener associated with the callback URL
+96 = The gatekeeper contact cannot be parsed
+97 = The job manager could not find the 'poe' command
+98 = The job manager could not find the 'mpirun' command
+99 = The provided RSL 'start_time' parameter is invalid"
+100 = The provided RSL 'reservation_handle' parameter is invalid
+
+101 = The provided RSL 'max_wall_time' parameter is invalid
+102 = The RSL 'max_wall_time' value is not zero or greater
+103 = The provided RSL 'max_cpu_time' parameter is invalid
+104 = The RSL 'max_cpu_time' value is not zero or greater
+105 = The job manager is misconfigured, a scheduler script is missing
+106 = The job manager is misconfigured, a scheduler script has invalid permissions
+107 = The job manager failed to signal the job
+108 = The job manager did not recognize/support the signal type
+109 = The job manager failed to get the job id from the local scheduler
+
+110 = The job manager is waiting for a commit signal
+111 = The job manager timed out while waiting for a commit signal
+112 = The provided RSL 'save_state' parameter is invalid
+113 = The provided RSL 'restart' parameter is invalid
+114 = The provided RSL 'two_phase' parameter is invalid
+115 = The RSL 'two_phase' value is not zero or greater
+116 = The provided RSL 'stdout_position' parameter is invalid
+117 = The RSL 'stdout_position' value is not zero or greater
+118 = The provided RSL 'stderr_position' parameter is invalid
+119 = The RSL 'stderr_position' value is not zero or greater
+
+120 = The job manager restart attempt failed
+121 = The job state file doesn't exist
+122 = Could not read the job state file
+123 = Could not write the job state file
+124 = The old job manager is still alive
+125 = The job manager state file TTL expired
+126 = It is unknown if the job was submitted
+127 = The provided RSL 'remote_io_url' parameter is invalid
+128 = Could not write the remote io url file
+129 = The standard output/error size is different
+
+130 = The job manager was sent a stop signal (job is still running)
+131 = The user proxy expired (job is still running)
+132 = The job was not submitted by original jobmanager
+133 = The job manager is not waiting for that commit signal
+134 = The provided RSL scheduler specific parameter is invalid
+135 = The job manager could not stage in a file
+136 = The scratch directory could not be created
+137 = The provided 'gass_cache' parameter is invalid
+138 = The RSL contains attributes which are not valid for job submission
+139 = The RSL contains attributes which are not valid for stdio update
+
+140 = The RSL contains attributes which are not valid for job restart
+141 = The provided RSL 'file_stage_in' parameter is invalid
+142 = The provided RSL 'file_stage_in_shared' parameter is invalid
+143 = The provided RSL 'file_stage_out' parameter is invalid
+144 = The provided RSL 'gass_cache' parameter is invalid
+145 = The provided RSL 'file_cleanup' parameter is invalid
+146 = The provided RSL 'scratch_dir' parameter is invalid
+147 = The provided scheduler-specific RSL parameter is invalid
+148 = A required RSL attribute was not defined in the RSL spec
+149 = The gass_cache attribute points to an invalid cache directory
+
+150 = The provided RSL 'save_state' parameter has an invalid value
+151 = The job manager could not open the RSL attribute validation file
+152 = The job manager could not read the RSL attribute validation file
+153 = The provided RSL 'proxy_timeout' is invalid
+154 = The RSL 'proxy_timeout' value is not greater than zero
+155 = The job manager could not stage out a file
+156 = The job contact string does not match any which the job manager is handling
+157 = Proxy delegation failed
+158 = The job manager could not lock the state lock file
+
+1000 = Failed to start up callback handler
+1003 = Job contact not set
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/AccessPolicy.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/AccessPolicy.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/AccessPolicy.json
new file mode 100644
index 0000000..8f6cfe1
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/AccessPolicy.json
@@ -0,0 +1,13 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/AccessPolicy.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Policy.json"}],
+ "properties": {
+ "EndpointID": {
+ "type": "string",
+ "description": "The ID of the Endpoint this AccessPolicy is for"
+ }
+ },
+ "required": ["EndpointID"]
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Activity.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Activity.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Activity.json
new file mode 100644
index 0000000..8bd2495
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Activity.json
@@ -0,0 +1,31 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Activity.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
+ "properties": {
+ "UserDomainID": {
+ "type": "string",
+ "description": "An ID"
+ },
+ "EndpointID": {
+ "type": "string",
+ "description": "The ID of the Endpoint managing Activity"
+ },
+ "ShareID": {
+ "type": "string",
+ "description": "The ID of the Share servicing this Activity"
+ },
+ "ResourceID": {
+ "type": "string",
+ "description": "The ID of the Resource executing this Activity"
+ },
+ "ActivityID": {
+ "type": "array",
+ "description": "The IDs of other Activities related to this one",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/AdminDomain.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/AdminDomain.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/AdminDomain.json
new file mode 100644
index 0000000..8ed4606
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/AdminDomain.json
@@ -0,0 +1,51 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/AdminDomain.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Domain.json"}],
+ "properties": {
+ "Distributed": {
+ "type": "boolean",
+ "description": "true if the services managed by the AdminDomain are geographically distributed"
+ },
+ "Owner": {
+ "type": "array",
+ "description": "Identification of persons or legal entities that own the resources in this AdminDomain",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ServiceID": {
+ "type": "array",
+ "description": "IDs of Services in this AdminDomain",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ChildDomainID": {
+ "type": "array",
+ "description": "IDs of AdminDomains aggregated by this AdminDomain",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ParentDomainID": {
+ "type": "string",
+ "description": "The ID of the AdminDomain that this AdminDomain participates in"
+ },
+ "ComputingServiceID": {
+ "type": "array",
+ "description": "IDs of ComputingServices in this AdminDomain",
+ "items": {
+ "type": "string"
+ }
+ },
+ "StorageServiceID": {
+ "type": "array",
+ "description": "IDs of StorageServices in this AdminDomain",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ApplicationEnvironment.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ApplicationEnvironment.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ApplicationEnvironment.json
new file mode 100644
index 0000000..89c78e0
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ApplicationEnvironment.json
@@ -0,0 +1,86 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ApplicationEnvironment.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
+ "properties": {
+ "AppName": {
+ "type": "string",
+ "description": "The name of the application"
+ },
+ "AppVersion": {
+ "type": "string",
+ "description": "The version of the application"
+ },
+ "State": {
+ "type": "string",
+ "description": "The current installation state of the application - AppEnvState_t"
+ },
+ "RemovalDate": {
+ "type": "string",
+ "description": "The date/time after which the application may be removed - DateTime_t"
+ },
+ "License": {
+ "type": "string",
+ "description": "The license under which the application is usable - License_t"
+ },
+ "Description": {
+ "type": "string",
+ "description": "A human-readable description of the application"
+ },
+ "BestBenchmark": {
+ "type": "array",
+ "description": "The type(s) of the benchmarks which best describe the sensitivity of this application to the performance of the ExecutionEnvironment - Benchmark_t",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ParallelSupport": {
+ "type": "string",
+ "description": "The type of supported parallel execution - ParallelSupport_t"
+ },
+ "MaxSlots": {
+ "type": "integer",
+ "description": "The maximum number of concurrent slots that may be used to run the application"
+ },
+ "MaxJobs": {
+ "type": "integer",
+ "description": "The maximum number of concurrent jobs that can run the application"
+ },
+ "MaxUserSeats": {
+ "type": "integer",
+ "description": "The maximum number of concurrent users that can run the application"
+ },
+ "FreeSlots": {
+ "type": "integer",
+ "description": "The maximum number slots currently available to run the application"
+ },
+ "FreeJobs": {
+ "type": "integer",
+ "description": "The maximum number of additional jobs that can run the application"
+ },
+ "FreeUserSeats": {
+ "type": "integer",
+ "description": "The maximum number of additional users that can run the application"
+ },
+ "ExecutionEnvironmentID": {
+ "type": "array",
+ "description": "ID(s) of ExecutionEnvironments where this ApplicationEnvironment can be used",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ComputingManagerID": {
+ "type": "string",
+ "description": "ID of the ComputingManager this ApplicationEnvironment is associated with"
+ },
+ "ApplicationHandleID": {
+ "type": "array",
+ "description": "ID(s) of the ApplicationHandles that can be used to refer to this environment",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "required": ["AppName","ComputingManagerID"]
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ApplicationHandle.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ApplicationHandle.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ApplicationHandle.json
new file mode 100644
index 0000000..e7972e9
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ApplicationHandle.json
@@ -0,0 +1,21 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ApplicationHandle.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
+ "properties": {
+ "Type": {
+ "type": "string",
+ "description": "The type of method used to set up an ApplicationEnvironment - ApplicationHandle_t (open enumeration)"
+ },
+ "Value": {
+ "type": "string",
+ "description": "How to set up the ApplicationEnvironment in the context of the Type"
+ },
+ "ApplicationEnvironmentID": {
+ "type": "string",
+ "description": "The ID of the ApplicationEnvironment this ApplicationHandle refers to"
+ }
+ },
+ "required": ["Type","Value","ApplicationEnvironmentID"]
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Benchmark.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Benchmark.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Benchmark.json
new file mode 100644
index 0000000..2b64261
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Benchmark.json
@@ -0,0 +1,21 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Benchmark.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
+ "properties": {
+ "Type": {
+ "type": "string",
+ "description": "The type of the benchmark - Benchmark_t (open enumeration)"
+ },
+ "Value": {
+ "type": "number",
+ "description": "The value of the benchmark"
+ },
+ "ComputingManagerID": {
+ "type": "string",
+ "description": "The ID of the ComputingManager this benchmark is for"
+ }
+ },
+ "required": ["Type","Value"]
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ComputingActivity.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ComputingActivity.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ComputingActivity.json
new file mode 100644
index 0000000..5fcae72
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ComputingActivity.json
@@ -0,0 +1,165 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ComputingActivity.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Activity.json"}],
+ "properties": {
+ "Type": {
+ "type": "string",
+ "description": "closed enumeration ComputingActivityType_t",
+ "enum": ["collectionelement","parallelelement","single","workflownode"]
+ },
+ "IDFromEndpoint": {
+ "type": "string",
+ "description": "The ID assigned by the ComputingEndpoint"
+ },
+ "LocalIDFromManager": {
+ "type": "string",
+ "description": "The local ID assigned by the ComputingManager"
+ },
+ "State": {
+ "type": "array",
+ "description": "open enumeration ComputingActivityState_t",
+ "items": {
+ "type": "string"
+ },
+ "minItems": 1
+ },
+ "RestartState": {
+ "type": "array",
+ "description": "open enumeration ComputingActivityState_t",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ExitCode": {
+ "type": "integer",
+ "description": "The exit code as returned by the main executable code or script of the job"
+ },
+ "ComputingManagerExitCode": {
+ "type": "string",
+ "description": "The exit code provided by the ComputingManager"
+ },
+ "Error": {
+ "type": "array",
+ "description": "The error messages as provided by the software components involved in the management of the job",
+ "items": {
+ "type": "string"
+ }
+ },
+ "WaitingPosition": {
+ "type": "integer",
+ "description": "The position of the job in the queue, if the job is waiting"
+ },
+ "Owner": {
+ "type": "string",
+ "description": "The Grid identity of the job's owner"
+ },
+ "LocalOwner": {
+ "type": "string",
+ "description": "The local user name of the job's owner"
+ },
+ "RequestedTotalWallTime": {
+ "type": "integer",
+ "description": "The total wall clock time requested by the job"
+ },
+ "RequestedTotalCPUTime": {
+ "type": "integer",
+ "description": "The total CPU time requested by the job"
+ },
+ "RequestedSlots": {
+ "type": "integer",
+ "description": "The number of slots requested for the job"
+ },
+ "RequestedApplicationEnvironment": {
+ "type": "array",
+ "description": "The AppName and Version of the requested ApplicationEnvironments",
+ "items": {
+ "type": "string"
+ }
+ },
+ "StdIn": {
+ "type": "string",
+ "description": "The name of the file used for standard input"
+ },
+ "StdOut": {
+ "type": "string",
+ "description": "The name of the file used for standard output"
+ },
+ "StdErr": {
+ "type": "string",
+ "description": "The name of the file used for standard error"
+ },
+ "LogDir": {
+ "type": "string",
+ "description": "The name of the directory which contains job logs"
+ },
+ "ExecutionNode": {
+ "type": "array",
+ "description": "Hostnames associated with the ExecutionEnvironments running the job",
+ "items": {
+ "type": "string"
+ }
+ },
+ "Queue": {
+ "type": "string",
+ "description": "The name of the ComputingManager queue that held the job before execution"
+ },
+ "UsedTotalWallTime": {
+ "type": "integer",
+ "description": "The total wall clock time consumed by the job so far (slots*seconds)"
+ },
+ "UsedTotalCpuTime": {
+ "type": "integer",
+ "description": "The total CPU time consumed by the job so far (seconds)"
+ },
+ "UsedMainMemory": {
+ "type": "integer",
+ "description": "The physical RAM currently used by the job (MB)"
+ },
+ "SubmissionTime": {
+ "type": "string",
+ "description": "The time when the job was submitted to the ComputingEndpoint (DateTime_t)"
+ },
+ "ComputingManagerSubmissionTime": {
+ "type": "string",
+ "description": "The time when the job was submitted to the ComputingManager (DateTime_t)"
+ },
+ "StartTime": {
+ "type": "string",
+ "description": "The time when the ComputingManager started the job (DateTime_t)"
+ },
+ "EndTime": {
+ "type": "string",
+ "description": "The time when the job ended in the Grid layer (DateTime_t)"
+ },
+ "ComputingManagerEndTime": {
+ "type": "string",
+ "description": "The time when the job ended according to the ComputingManager (DateTime_t)"
+ },
+ "WorkingAreaEraseTime": {
+ "type": "string",
+ "description": "The time when working area will be removed from storage (DateTime_t)"
+ },
+ "ProxyExpirationTime": {
+ "type": "string",
+ "description": "The expiration time of the Grid proxy associated with the job (DateTime_t)"
+ },
+ "SubmissionHost": {
+ "type": "string",
+ "description": "The name of the host from which the job was submitted"
+ },
+ "SubmissionClientName": {
+ "type": "string",
+ "description": "The name of the software client used to submit the job"
+ },
+ "OtherMessages": {
+ "type": "array",
+ "description": "Optional messages provided by either the Grid layer or the ComputingManager",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "required": ["State","Owner"]
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ComputingEndpoint.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ComputingEndpoint.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ComputingEndpoint.json
new file mode 100644
index 0000000..f94f889
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ComputingEndpoint.json
@@ -0,0 +1,44 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ComputingEndpoint.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Endpoint.json"}],
+ "properties": {
+ "Staging": {
+ "type": "string",
+ "description": "Supported file staging functionality - Staging_t",
+ "enum": ["none","stagingin","staginginout","stagingout"]
+ },
+ "JobDescription": {
+ "type": "array",
+ "description": "Supported job description languages - JobDescription_t (open Enumeration)",
+ "items": {
+ "type": "string"
+ }
+ },
+ "TotalJobs": {
+ "type": "integer",
+ "description": "The total number of Grid jobs known to the system"
+ },
+ "RunningJobs": {
+ "type": "integer",
+ "description": "The number of Grid jobs which are running in an ExecutionEnvironment"
+ },
+ "WaitingJobs": {
+ "type": "integer",
+ "description": "The number of Grid jobs which are waiting to start executing"
+ },
+ "StagingJobs": {
+ "type": "integer",
+ "description": "The number of Grid jobs staging files before or after execution"
+ },
+ "SuspendedJobs": {
+ "type": "integer",
+ "description": "The number of Grid jobs that started to execute, but are now suspended"
+ },
+ "PreLRMSWaitingJobs": {
+ "type": "integer",
+ "description": "The number of Grid jobs managed by the Grid software, but not yet passed to the LRMS"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ComputingManager.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ComputingManager.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ComputingManager.json
new file mode 100644
index 0000000..aecb114
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ComputingManager.json
@@ -0,0 +1,117 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ComputingManager.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Share.json"}],
+ "properties": {
+ "Reservation": {
+ "type": "boolean",
+ "description": "Whether advance reservation is supported (no value implies undefined in ExtendedBoolean_t)"
+ },
+ "BulkSubmission": {
+ "type": "boolean",
+ "description": "Whether multiple jobs can be submitted at once (no value implies undefined in ExtendedBoolean_t)"
+ },
+ "TotalPhysicalCPUs": {
+ "type": "integer",
+ "description": "The total number of physical CPUs managed by this ComputingManager"
+ },
+ "TotalLogicalCPUs": {
+ "type": "integer",
+ "description": "The total number of logical CPUs managed by this ComputingManager"
+ },
+ "TotalSlots": {
+ "type": "integer",
+ "description": "The total number of slots managed by this ComputingManager"
+ },
+ "SlotsUsedByLocalJobs": {
+ "type": "integer",
+ "description": "The number of slots currently used by jobs submitted via a non-Grid interface"
+ },
+ "SlotsUsedByGridJobs": {
+ "type": "integer",
+ "description": "The number of slots currently used by jobs submitted via a non-Grid interface"
+ },
+ "Homogeneous": {
+ "type": "boolean",
+ "description": "Whether this ComputingManager manages only one type of ExecutionEnvironment"
+ },
+ "NetworkInfo": {
+ "type": "array",
+ "description": "The types of internal network connections between ExecutionEnvironments (NetworkInfo_t)",
+ "items": {
+ "type": "string"
+ }
+ },
+ "LocalCPUDistribution": {
+ "type": "boolean",
+ "description": "Classification of the managed ExecutionEnvironments aggregated by the number of logical CPUs"
+ },
+ "WorkingAreaShared": {
+ "type": "boolean",
+ "description": "True if the working area is shared across different ExecutionEnvironments"
+ },
+ "WorkingAreaGuaranteed": {
+ "type": "boolean",
+ "description": "True if the job is guaranteed all of WorkingAreaTotal"
+ },
+ "WorkingAreaTotal": {
+ "type": "integer",
+ "description": "Total size of the working area available to single slot jobs (GB)"
+ },
+ "WorkingAreaFree": {
+ "type": "integer",
+ "description": "The amount of free space in the working area (GB)"
+ },
+ "WorkingAreaLifeTime": {
+ "type": "integer",
+ "description": "The minimum guaranteed lifetime of files created in the working area (seconds)"
+ },
+ "WorkingAreaMultiSlotTotal": {
+ "type": "integer",
+ "description": "The total size of the working area across all ExecutionEnvironments (GB)"
+ },
+ "WorkingAreaMultiSlotFree": {
+ "type": "integer",
+ "description": "The available space in the working area across all ExecutionEnvironments (GB)"
+ },
+ "WorkingAreaMultiSlotLifeTime": {
+ "type": "integer",
+ "description": "The minimum guaranteed lifetime of files created in the working area (seconds)"
+ },
+ "CacheTotal": {
+ "type": "integer",
+ "description": "If local caching of input files is supported, the total size of the area they may be stored in"
+ },
+ "CacheFree": {
+ "type": "integer",
+ "description": "If local caching of input files is supported, the available size of the area they may be stored in"
+ },
+ "TmpDir": {
+ "type": "string",
+ "description": "The absolute path of a temporary directory local to an ExecutionEnvironment"
+ },
+ "ScratchDir": {
+ "type": "string",
+ "description": "The absolute path of a shared directory available for application data"
+ },
+ "ApplicationDir": {
+ "type": "string",
+ "description": "The absolute path of a directory available for installation of persistent application software"
+ },
+ "ApplicationEnvironmentID": {
+ "type": "array",
+ "description": "ID(s) of ApplicationEnvironments provided by this ComputingManager",
+ "items": {
+ "type": "string"
+ }
+ },
+ "BenchmarkID": {
+ "type": "array",
+ "description": "ID(s) of Benchmarks associated with this ComputingManager",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ComputingService.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ComputingService.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ComputingService.json
new file mode 100644
index 0000000..9cfde1b
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ComputingService.json
@@ -0,0 +1,32 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ComputingService.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Service.json"}],
+ "properties": {
+ "TotalJobs": {
+ "type": "integer",
+ "description": "The total number of Grid jobs known to the system"
+ },
+ "RunningJobs": {
+ "type": "integer",
+ "description": "The number of Grid jobs which are running in an ExecutionEnvironment"
+ },
+ "WaitingJobs": {
+ "type": "integer",
+ "description": "The number of Grid jobs which are waiting to start executing"
+ },
+ "StagingJobs": {
+ "type": "integer",
+ "description": "The number of Grid jobs staging files before or after execution"
+ },
+ "SuspendedJobs": {
+ "type": "integer",
+ "description": "The number of Grid jobs that started to execute, but are now suspended"
+ },
+ "PreLRMSWaitingJobs": {
+ "type": "integer",
+ "description": "The number of Grid jobs managed by the Grid software, but not yet passed to the LRMS"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ComputingShare.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ComputingShare.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ComputingShare.json
new file mode 100644
index 0000000..340c83e
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ComputingShare.json
@@ -0,0 +1,182 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ComputingShare.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Share.json"}],
+ "properties": {
+ "MappingQueue": {
+ "type": "string",
+ "description": "The name of the queue in the LRMS where jobs in this share are submitted"
+ },
+ "MaxWallTime": {
+ "type": "integer",
+ "description": "The maximum wall clock time that a single-slot job can run (seconds)"
+ },
+ "MaxMultiSlotWallTime": {
+ "type": "integer",
+ "description": "The maximum wall clock time that a multi-slot job can run (seconds)"
+ },
+ "DefaultWallTime": {
+ "type": "integer",
+ "description": "The default wall clock per slot assumed by the LRMS if a maximum time is not specified (seconds)"
+ },
+ "MaxCPUTime": {
+ "type": "integer",
+ "description": "The maximum pre-slot CPU time that a job can request (seconds)"
+ },
+ "MaxTotalCPUTime": {
+ "type": "integer",
+ "description": "The maximum amount of CPU time that a job can request across all slots assigned to it (seconds)"
+ },
+ "MinCPUTime": {
+ "type": "integer",
+ "description": "The minimum pre-slot CPU time that a job can request (seconds)"
+ },
+ "DefaultCPUTime": {
+ "type": "integer",
+ "description": "The default CPU time limit assumed by the LRMS if a maximum time is not specified (seconds)"
+ },
+ "MaxTotalJobs": {
+ "type": "integer",
+ "description": "The maximum number of jobs that can be in this Share"
+ },
+ "MaxRunningJobs": {
+ "type": "integer",
+ "description": "The maximum number of jobs that can be running in this Share"
+ },
+ "MaxWaitingJobs": {
+ "type": "integer",
+ "description": "The maximum number of jobs that can be waiting in this Share"
+ },
+ "MaxPreLRMSWaitingJobs": {
+ "type": "integer",
+ "description": "The maximum number of jobs that can be waiting in the Grid layer for this Share"
+ },
+ "MaxUserRunningJobs": {
+ "type": "integer",
+ "description": "The maximum number of jobs that can be running in this Share per user"
+ },
+ "MaxSlotsPerJob": {
+ "type": "integer",
+ "description": "The maximum number of slots that can be allocated to a single job in this Share"
+ },
+ "MaxStageInStreams": {
+ "type": "integer",
+ "description": "The maximum number of streams available to stage files in"
+ },
+ "MaxStageOutStreams": {
+ "type": "integer",
+ "description": "The maximum number of streams available to stage files out"
+ },
+ "ScheduingPolicy": {
+ "type": "string",
+ "description": "The scheduling policy used by the share - SchedulingPolicy_t (open enumeration)"
+ },
+ "MaxMainMemory": {
+ "type": "integer",
+ "description": "The maximum amount of physical RAM that a job can use (MB)"
+ },
+ "GuaranteedMainMemory": {
+ "type": "integer",
+ "description": "The amount of physical RAM that a job will have available (MB)"
+ },
+ "MaxVirtualMemory": {
+ "type": "integer",
+ "description": "The maximum amount memory (RAM+swap) that a job can use (MB)"
+ },
+ "GuaranteedVirtualMemory": {
+ "type": "integer",
+ "description": "The amount of memory (RAM+swap) that a job will have available (MB)"
+ },
+ "MaxDiskSpace": {
+ "type": "integer",
+ "description": "The maximum disk space that a job can use in the working area (GB)"
+ },
+ "DefaultStorageServiceID": {
+ "type": "string",
+ "description": "The ID of the default StorageService used to store files"
+ },
+ "Preemption": {
+ "type": "boolean",
+ "description": "Whether jobs can be preempted and resumed (no value implies undefined in ExtendedBoolean_t)"
+ },
+ "ServingState": {
+ "type": "string",
+ "description": "How the Share is currently serving jobs",
+ "enum": ["closed","draining","production","queueing"]
+ },
+ "TotalJobs": {
+ "type": "integer",
+ "description": "The total number of jobs in any state"
+ },
+ "RunningJobs": {
+ "type": "integer",
+ "description": "The number of running jobs submitted through Grid or non-Grid interfaces"
+ },
+ "LocalRunningJobs": {
+ "type": "integer",
+ "description": "The number of running jobs submitted using non-Grid interfaces"
+ },
+ "WaitingJobs": {
+ "type": "integer",
+ "description": "The number of waiting jobs submitted through Grid or non-Grid interfaces"
+ },
+ "LocalWaitingJobs": {
+ "type": "integer",
+ "description": "The number of waiting jobs submitted using non-Grid interfaces"
+ },
+ "SuspendedJobs": {
+ "type": "integer",
+ "description": "The number of suspended jobs submitted through Grid or non-Grid interfaces"
+ },
+ "LocalSuspendedJobs": {
+ "type": "integer",
+ "description": "The number of suspended jobs submitted using non-Grid interfaces"
+ },
+ "StagingJobs": {
+ "type": "integer",
+ "description": "The number of jobs staging files before or after execution"
+ },
+ "PreLRMSWaitingJobs": {
+ "type": "integer",
+ "description": "The number of Grid jobs which have not yet been passed to the LRMS"
+ },
+ "EstimatedAverageWaitingTime": {
+ "type": "integer",
+ "description": "An estimate of the average time a job will wait before it starts to execute (seconds)"
+ },
+ "EstimatedWorstWaitingTime": {
+ "type": "integer",
+ "description": "An estimate of the worst-case time a job will wait before it starts to execute (seconds)"
+ },
+ "FreeSlots": {
+ "type": "integer",
+ "description": "The number of slots which are currently available for use"
+ },
+ "FreeSlotsWithDuration": {
+ "type": "string",
+ "description": "The number of slots which are currently available for use and how long they are available"
+ },
+ "UsedSlots": {
+ "type": "integer",
+ "description": "The number of slots currently in use"
+ },
+ "RequestedSlots": {
+ "type": "integer",
+ "description": "The number of slots needd to execute all waiting and staging jobs"
+ },
+ "ReservationPolicy": {
+ "type": "string",
+ "description": "The policy used for advance reservation - ReservationPolicy_t",
+ "enum": ["mandatory","none","optional"]
+ },
+ "Tag": {
+ "type": "array",
+ "description": "UserDomain-defined tags for this Share",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "required": ["ServingState"]
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Contact.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Contact.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Contact.json
new file mode 100644
index 0000000..436b262
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Contact.json
@@ -0,0 +1,32 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Contact.json",
+ "description": "A GLUE 2 Contact",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
+ "properties": {
+ "Detail": {
+ "type": "string",
+ "description": "A URI embedding the contact information"
+ },
+ "Type": {
+ "type": "string",
+ "description": "closed enumeration ContactType_t",
+ "enum": ["general","security","sysadmin","usersupport"]
+ },
+ "ServiceID": {
+ "type": "array",
+ "description": "The IDs of Services associated with this Contact",
+ "items": {
+ "type": "string"
+ }
+ },
+ "DomainID": {
+ "type": "array",
+ "description": "The IDs of Domains associated with this Contact",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/DataStore.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/DataStore.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/DataStore.json
new file mode 100644
index 0000000..8f15447
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/DataStore.json
@@ -0,0 +1,30 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/DataStore.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Resource.json"}],
+ "properties": {
+ "Type": {
+ "type": "string",
+ "description": "The type of storage medium - DataStoreType_t (disk,optical,tape,...)"
+ },
+ "Latency": {
+ "type": "string",
+ "description": "The latency category under normal operating conditions - AccessLatency_t",
+ "enum": ["nearline","offline","online"]
+ },
+ "TotalSize": {
+ "type": "integer",
+ "description": "The total amount of storage (GB)"
+ },
+ "FreeSize": {
+ "type": "integer",
+ "description": "The amount of available storage (GB)"
+ },
+ "UsedSize": {
+ "type": "integer",
+ "description": "The amount of used storage (GB)"
+ }
+ },
+ "required": ["Type","Latency"]
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Domain.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Domain.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Domain.json
new file mode 100644
index 0000000..5bd996b
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Domain.json
@@ -0,0 +1,30 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Domain.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
+ "properties": {
+ "Description": {
+ "type": "string",
+ "description": "A description of the Domain"
+ },
+ "WWW": {
+ "type": "array",
+ "description": "URLs of web pages with more information about the Domain",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ContactID": {
+ "type": "array",
+ "description": "IDs of Contacts for this Domain",
+ "items": {
+ "type": "string"
+ }
+ },
+ "LocationID": {
+ "type": "string",
+ "description": "The ID of the primary Location for this Domain"
+ }
+ }
+}
[4/7] airavata git commit: Add new gfac email monitor module to gfac
monitor module and Intorduced gfac-hpc-monitor module to keep all hoc-monitor
code
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Endpoint.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Endpoint.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Endpoint.json
new file mode 100644
index 0000000..b75b02a
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Endpoint.json
@@ -0,0 +1,147 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Endpoint.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
+ "properties": {
+ "URL": {
+ "type": "string",
+ "description": "Network location of the endpoint"
+ },
+ "Capability": {
+ "type": "array",
+ "description": "Capability_t (open enumeration)",
+ "items": {
+ "type": "string"
+ }
+ },
+ "Technology": {
+ "type": "string",
+ "description": "EndpointTechnology_t"
+ },
+ "InterfaceName": {
+ "type": "string",
+ "description": "InterfaceName_t"
+ },
+ "InterfaceVersion": {
+ "type": "string",
+ "description": "The version of the primary interface protocol (free format)"
+ },
+ "InterfaceExtension": {
+ "type": "array",
+ "description": "URIs identifying supported extensions to the interface protocol",
+ "items": {
+ "type": "string"
+ }
+ },
+ "WSDL": {
+ "type": "array",
+ "description": "URLs of WSDL document(s) describing the interface",
+ "items": {
+ "type": "string"
+ }
+ },
+ "SupportedProfile": {
+ "type": "array",
+ "description": "URI(s) identifying supported profiles for the Endpoint",
+ "items": {
+ "type": "string"
+ }
+ },
+ "Semantics": {
+ "type": "array",
+ "description": "URL(s) of documents providing human-readable descriptions of the semantics of the Endpoint",
+ "items": {
+ "type": "string"
+ }
+ },
+ "Implementor": {
+ "type": "string",
+ "description": "The name of the main organization implementing the Endpoint"
+ },
+ "ImplementationName": {
+ "type": "string",
+ "description": "The name of the implementation of the Endpoint"
+ },
+ "ImplementationVersion": {
+ "type": "string",
+ "description": "The version of the implementation of the Endpoint"
+ },
+ "QualityLevel": {
+ "type": "string",
+ "description": "QualityLevel_t",
+ "enum": ["development","pre-production","production","testing"]
+ },
+ "HealthState": {
+ "type": "string",
+ "description": "The operational status of the Endpoint",
+ "enum": ["critical","ok","other","unknown","warning"]
+ },
+ "HealthStateInfo": {
+ "type": "string",
+ "description": "A human-readable explanation of the HealthState of this Endpoint"
+ },
+ "ServingState": {
+ "type": "string",
+ "description": "If the endpoint is accepting and serving requests",
+ "enum": ["closed","draining","production","queueing"]
+ },
+ "StartTime": {
+ "type": "string",
+ "description": "The start time of the Service associated with this Endpoint (DateTime_t)"
+ },
+ "IssuerCA": {
+ "type": "string",
+ "description": "The DN of the CA issuing the certificate presented by this Endpoint"
+ },
+ "TrustedCA": {
+ "type": "array",
+ "description": "DN(s) of CAs trusted by this Endpoint",
+ "items": {
+ "type": "string"
+ }
+ },
+ "DowntimeAnnounce": {
+ "type": "string",
+ "description": "When the next scheduled downtime was announced (DateTime_t)"
+ },
+ "DowntimeStart": {
+ "type": "string",
+ "description": "When the next scheduled downtime will start (DateTime_t)"
+ },
+ "DowntimeEnd": {
+ "type": "string",
+ "description": "When the next scheduled downtime will end (DateTime_t)"
+ },
+ "DowntimeInfo": {
+ "type": "string",
+ "description": "Human-readable of the next scheduled downtime"
+ },
+ "ServiceID": {
+ "type": "string",
+ "description": "The ID of the Service associated with this Endpoint"
+ },
+ "ShareID": {
+ "type": "array",
+ "description": "The IDs of the Shares accessible from this Endpoint",
+ "items": {
+ "type": "string"
+ }
+ },
+ "AccessPolicyID": {
+ "type": "array",
+ "description": "IDs of AccessPolicies associated with this Endpoint",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ActivityID": {
+ "type": "array",
+ "description": "IDs of Activities being managed through this Endpoint",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "required": ["InterfaceName","QualityLevel","HealthState","ServingState","ServiceID"]
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Entity.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Entity.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Entity.json
new file mode 100644
index 0000000..5d1ae46
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Entity.json
@@ -0,0 +1,35 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json",
+ "type": "object",
+ "properties": {
+ "CreationTime": {
+ "type": "string",
+ "description": "The creation time of this entity in the format: CCYY-MM-DDThh:mm:ss[Z|(+|-)hh:mm]"
+ },
+ "Validity": {
+ "type": "integer",
+ "description": "The number of seconds after CreationTime that this entity should be considered relevant"
+ },
+ "ID": {
+ "type": "string",
+ "description": "A globally unique identifier for this entity"
+ },
+ "Name": {
+ "type": "string",
+ "description": "A human-readable name"
+ },
+ "OtherInfo": {
+ "type": "array",
+ "description": "Placeholder for information that does not fit in any other attribute",
+ "items": {
+ "type": "string"
+ }
+ },
+ "Extension": {
+ "type": "object",
+ "description": "Key/value pairs enabling the association of extra information not captured by the model"
+ }
+ },
+ "required": ["ID"]
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ExecutionEnvironment.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ExecutionEnvironment.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ExecutionEnvironment.json
new file mode 100644
index 0000000..77bf876
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ExecutionEnvironment.json
@@ -0,0 +1,115 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ExecutionEnvironment.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Resource.json"}],
+ "properties": {
+ "Platform": {
+ "type": "string",
+ "description": "The platform architecture - Platform_t (open enumeration)"
+ },
+ "VirtualMachine": {
+ "type": "boolean",
+ "description": "True if the ExecutionEnvironment is a virtual machine"
+ },
+ "TotalInstances": {
+ "type": "integer",
+ "description": "The total number of ExecutionEnvironment instances"
+ },
+ "UsedInstances": {
+ "type": "integer",
+ "description": "The number of ExecutionEnvironment instances in use"
+ },
+ "UnavailableInstances": {
+ "type": "integer",
+ "description": "The number of ExecutionEnvironment instances that are unavailable"
+ },
+ "PhysicalCPUs": {
+ "type": "integer",
+ "description": "The number of physical CPUs in one ExecutionEnvironment instance"
+ },
+ "LogicalCPUs": {
+ "type": "integer",
+ "description": "The number of logical CPUs in one ExecutionEnvironment instance"
+ },
+ "CPUMultiplicity": {
+ "type": "string",
+ "description": "Information about the CPUs and cores in an execution environment",
+ "enum": ["multicpu-multicore","multicpu-singlecore","singlecpu-multicore","singlecpu-singlecore"]
+ },
+ "CPUVendor": {
+ "type": "string",
+ "description": "The name of the manufacturer of the CPU"
+ },
+ "CPUModel": {
+ "type": "string",
+ "description": "The model of the CPU, as defined by the vendor"
+ },
+ "CPUVersion": {
+ "type": "string",
+ "description": "The specific version name of the CPU, as defined by the vendor"
+ },
+ "CPUClockSpeed": {
+ "type": "integer",
+ "description": "The clock speed of the CPU (MHz)"
+ },
+ "CPUTimeScalingFactor": {
+ "type": "float",
+ "description": "The factor used by the ComputingManager to scale the CPU time limit"
+ },
+ "WallTimeScalingFactor": {
+ "type": "float",
+ "description": "The factor used by the ComputingManager to scale the wallclock time limit"
+ },
+ "MainMemorySize": {
+ "type": "integer",
+ "description": "The total amount of physical RAM in one ExecutionEnvironment instance (MB)"
+ },
+ "VirtualMemorySize": {
+ "type": "integer",
+ "description": "The total amount of virtual memory (RAM+swap) in one ExecutionEnvironment instance (MB)"
+ },
+ "OSFamily": {
+ "type": "string",
+ "description": "The general family of the operating system - OSFamily_t (open enumeration)"
+ },
+ "OSName": {
+ "type": "string",
+ "description": "The specific name of the operating system - OSName_t (open enumeration)"
+ },
+ "OSVersion": {
+ "type": "string",
+ "description": "The version of the operating system, as defined by the vendor"
+ },
+ "ConnectivityIn": {
+ "type": "boolean",
+ "description": "True if direct inbound network connectiity is available to a running job"
+ },
+ "ConnectivityOut": {
+ "type": "boolean",
+ "description": "True if direct outbound network connectiity is available to a running job"
+ },
+ "NetworkInfo": {
+ "type": "array",
+ "description": "The types of internal network connections between ExecutionEnvironments - NetworkInfo_t (open enumeration)",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ApplicationEnvironmentID": {
+ "type": "array",
+ "description": "ID(s) of ApplicationEnvironments available in this ExecutionEnvironment",
+ "items": {
+ "type": "string"
+ }
+ },
+ "BenchmarkID": {
+ "type": "array",
+ "description": "ID(s) of Benchmarks associated with this ExecutionEnvironment",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "required": ["Platform","MainMemorySize","OSFamily","ConnectivityIn","ConnectivityOut"]
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Glue2.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Glue2.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Glue2.json
new file mode 100644
index 0000000..bb80505
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Glue2.json
@@ -0,0 +1,246 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Glue2.json",
+ "description": "A GLUE 2 document",
+ "type": "object",
+ "properties": {
+ "Entity": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}]
+ }
+ },
+ "Location": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Location.json"}]
+ }
+ },
+ "Contact": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Contact.json"}]
+ }
+ },
+ "Domain": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Domain.json"}]
+ }
+ },
+ "AdminDomain": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/AdminDomain.json"}]
+ }
+ },
+ "UserDomain": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/UserDomain.json"}]
+ }
+ },
+ "Service": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Service.json"}]
+ }
+ },
+ "Endpoint": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Endpoint.json"}]
+ }
+ },
+ "Share": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Share.json"}]
+ }
+ },
+ "Manager": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Manager.json"}]
+ }
+ },
+ "Resource": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Resource.json"}]
+ }
+ },
+ "Activity": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Activity.json"}]
+ }
+ },
+ "Policy": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Policy.json"}]
+ }
+ },
+ "AccessPolicy": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/AccessPolicy.json"}]
+ }
+ },
+ "MappingPolicy": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/MappingPolicy.json"}]
+ }
+ },
+ "ComputingService": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ComputingService.json"}]
+ }
+ },
+ "ComputingEndpoint": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ComputingEndpoint.json"}]
+ }
+ },
+ "ComputingShare": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ComputingShare.json"}]
+ }
+ },
+ "ComputingManager": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ComputingManager.json"}]
+ }
+ },
+ "Benchmark": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Benchmark.json"}]
+ }
+ },
+ "ExecutionEnvironment": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ExecutionEnvironment.json"}]
+ }
+ },
+ "ApplicationEnvironment": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ApplicationEnvironment.json"}]
+ }
+ },
+ "ApplicationHandle": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ApplicationHandle.json"}]
+ }
+ },
+ "ComputingActivity": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ComputingActivity.json"}]
+ }
+ },
+ "ToStorageService": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ToStorageService.json"}]
+ }
+ },
+ "StorageService": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageService.json"}]
+ }
+ },
+ "StorageServiceCapacity": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageServiceCapacity.json"}]
+ }
+ },
+ "StorageAccessProtocol": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageAccessProtocol.json"}]
+ }
+ },
+ "StorageEndpoint": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageEndpoint.json"}]
+ }
+ },
+ "StorageShare": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageShare.json"}]
+ }
+ },
+ "StorageShareCapacity": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageShareCapacity.json"}]
+ }
+ },
+ "StorageManager": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageManager.json"}]
+ }
+ },
+ "DataStore": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/DataStore.json"}]
+ }
+ },
+ "ToComputingService": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ToComputingService.json"}]
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Location.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Location.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Location.json
new file mode 100644
index 0000000..8491cc0
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Location.json
@@ -0,0 +1,47 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Location.json",
+ "description": "A GLUE 2 Location",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
+ "properties": {
+ "Address": {
+ "type": "string",
+ "description": "A free format street address"
+ },
+ "Place": {
+ "type": "string",
+ "description": "Name of town/city"
+ },
+ "Country": {
+ "type": "string",
+ "description": "Name of country"
+ },
+ "PostalCode": {
+ "type": "string",
+ "description": "Postal code"
+ },
+ "Latitude": {
+ "type": "number",
+ "description": "Position north (positive) or south (negative) of the equator in degrees"
+ },
+ "Longitude": {
+ "type": "number",
+ "description": "Position east (positive) or west (negative) of the primary meridian in degrees"
+ },
+ "ServiceID": {
+ "type": "array",
+ "description": "The IDs of Services at this location",
+ "items": {
+ "type": "string"
+ }
+ },
+ "DomainID": {
+ "type": "array",
+ "description": "The IDs of Domains at this location",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Manager.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Manager.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Manager.json
new file mode 100644
index 0000000..d1df50a
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Manager.json
@@ -0,0 +1,28 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Manager.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
+ "properties": {
+ "ProductName": {
+ "type": "string",
+ "description": "The name of the software product which implements the Manager"
+ },
+ "ProductVersion": {
+ "type": "string",
+ "description": "The version of the software product which implements the Manager"
+ },
+ "ServiceID": {
+ "type": "string",
+ "description": "The ID of the Service this Share participates in"
+ },
+ "ResourceID": {
+ "type": "array",
+ "description": "ID(s) of Resources associated with this Share",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "required": ["ProductName","ServiceID","ResourceID"]
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/MappingPolicy.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/MappingPolicy.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/MappingPolicy.json
new file mode 100644
index 0000000..268844d
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/MappingPolicy.json
@@ -0,0 +1,13 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/MappingPolicy.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Policy.json"}],
+ "properties": {
+ "ShareID": {
+ "type": "string",
+ "description": "The ID of the Share this MappingPolicy is for"
+ }
+ },
+ "required": ["ShareID"]
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Policy.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Policy.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Policy.json
new file mode 100644
index 0000000..f936699
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Policy.json
@@ -0,0 +1,27 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Policy.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
+ "properties": {
+ "Scheme": {
+ "type": "string",
+ "description": "PolicyScheme_t (open enumeration)"
+ },
+ "Rule": {
+ "type": "array",
+ "description": "Policy rules",
+ "items": {
+ "type": "string"
+ }
+ },
+ "UserDomainID": {
+ "type": "array",
+ "description": "The ID(s) of the UserDomains this Policy applies to",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "required": ["Scheme","Rule","UserDomainID"]
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Resource.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Resource.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Resource.json
new file mode 100644
index 0000000..88d08ad
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Resource.json
@@ -0,0 +1,27 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Resource.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
+ "properties": {
+ "ManagerID": {
+ "type": "string",
+ "description": "The ID of the Manager for this Resource"
+ },
+ "ShareID": {
+ "type": "array",
+ "description": "The ID(s) of the Shares this Resource is part of",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ActivityID": {
+ "type": "array",
+ "description": "The ID(s) of Activities consuming from this Share",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "required": ["ManagerID"]
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Service.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Service.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Service.json
new file mode 100644
index 0000000..4662407
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Service.json
@@ -0,0 +1,75 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Service.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
+ "properties": {
+ "Capability": {
+ "type": "array",
+ "description": "Capability_t (open enumeration)",
+ "items": {
+ "type": "string"
+ }
+ },
+ "Type": {
+ "type": "string",
+ "description": "ServiceType_t (open enumeration)"
+ },
+ "QualityLevel": {
+ "type": "string",
+ "description": "QualityLevel_t",
+ "enum": ["development","pre-production","production","testing"]
+ },
+ "StatusInfo": {
+ "type": "array",
+ "description": "URLs of web pages providing additional information",
+ "items": {
+ "type": "string"
+ }
+ },
+ "Complexity": {
+ "type": "string",
+ "description": "A human-readable description of the number of endpoint types, shares, and resources"
+ },
+ "EndpointID": {
+ "type": "array",
+ "description": "The IDs of Endpoints for this Service",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ShareID": {
+ "type": "array",
+ "description": "The IDs of the Shares offered by this Service",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ManagerID": {
+ "type": "array",
+ "description": "The IDs of the Managers of this Service",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ContactID": {
+ "type": "array",
+ "description": "The IDs of Contacts for this Service",
+ "items": {
+ "type": "string"
+ }
+ },
+ "LocationID": {
+ "type": "string",
+ "description": "The ID of the primary Location of this Service"
+ },
+ "ServiceID": {
+ "type": "array",
+ "description": "The IDs of Services related to this Service",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "required": ["Type","QualityLevel"]
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Share.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Share.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Share.json
new file mode 100644
index 0000000..258fc1b
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/Share.json
@@ -0,0 +1,45 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Share.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
+ "properties": {
+ "Description": {
+ "type": "string",
+ "description": "A human-readable description of the Share"
+ },
+ "EndpointID": {
+ "type": "array",
+ "description": "The ID(s) of the Endpoints that can be used to access this Share",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ResourceID": {
+ "type": "array",
+ "description": "The ID(s) of the Resources associated with this Share",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ServiceID": {
+ "type": "string",
+ "description": "The ID of the Service this Share participates in"
+ },
+ "ActivityID": {
+ "type": "array",
+ "description": "The ID(s) of Activities consuming from this Share",
+ "items": {
+ "type": "string"
+ }
+ },
+ "MappingPolicyID": {
+ "type": "array",
+ "description": "ID(s) of MappingPolicies associated with this Share",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "required": ["ServiceID"]
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageAccessProtocol.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageAccessProtocol.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageAccessProtocol.json
new file mode 100644
index 0000000..05a830b
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageAccessProtocol.json
@@ -0,0 +1,32 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageAccessProtocol.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
+ "properties": {
+ "Type": {
+ "type": "string",
+ "description": "The type of the protocol - StorageAccessProtocol_t"
+ },
+ "Version": {
+ "type": "string",
+ "description": "The version of the protocol supported"
+ },
+ "MaxStreams": {
+ "type": "integer",
+ "description": "The maximum number of parallel network streams which can be usef for a single transfer"
+ },
+ "StorageServiceID": {
+ "type": "string",
+ "description": "The ID of the StorageService this protocol is available for"
+ },
+ "ToComputingServiceID": {
+ "type": "array",
+ "description": "The ID(s) ToComputingService objects that describe connectivity to ComputingServices",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "required": ["Type","Version","StorageServiceID"]
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageEndpoint.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageEndpoint.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageEndpoint.json
new file mode 100644
index 0000000..38b27c4
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageEndpoint.json
@@ -0,0 +1,8 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageEndpoint.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Endpoint.json"}],
+ "properties": {
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageManager.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageManager.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageManager.json
new file mode 100644
index 0000000..f3984f6
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageManager.json
@@ -0,0 +1,8 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageManager.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Share.json"}],
+ "properties": {
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageService.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageService.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageService.json
new file mode 100644
index 0000000..a03d111
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageService.json
@@ -0,0 +1,22 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageService.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Service.json"}],
+ "properties": {
+ "StorageAccessProtocolID": {
+ "type": "array",
+ "description": "The ID(s) of the StorageAccessProtocols supported by this service",
+ "items": {
+ "type": "string"
+ }
+ },
+ "StorageServiceCapacityID": {
+ "type": "array",
+ "description": "The ID(s) of the StorageServiceCapacities for this Service",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageServiceCapacity.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageServiceCapacity.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageServiceCapacity.json
new file mode 100644
index 0000000..a25c204
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageServiceCapacity.json
@@ -0,0 +1,33 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageServiceCapacity.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
+ "properties": {
+ "Type": {
+ "type": "string",
+ "description": "The type of storage capacity - StorageCapacity_t"
+ },
+ "TotalSize": {
+ "type": "integer",
+ "description": "The total amount of storage of this type (GB)"
+ },
+ "FreeSize": {
+ "type": "integer",
+ "description": "The amount of currently available storage of this type (GB)"
+ },
+ "UsedSize": {
+ "type": "integer",
+ "description": "The amount storage of this type in use (GB)"
+ },
+ "ReservedSize": {
+ "type": "integer",
+ "description": "The amount storage of this type which is not in use, but has been reserved for use in use (GB)"
+ },
+ "StorageServiceID": {
+ "type": "string",
+ "description": "The ID of the StorageService this capacity describes"
+ }
+ },
+ "required": ["Type","StorageServiceID"]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageShare.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageShare.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageShare.json
new file mode 100644
index 0000000..9703118
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageShare.json
@@ -0,0 +1,65 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageShare.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Share.json"}],
+ "properties": {
+ "ServingState": {
+ "type": "string",
+ "description": "How the Share is currently serving jobs",
+ "enum": ["closed","draining","production","queueing"]
+ },
+ "Path": {
+ "type": "string",
+ "description": "A default namespace where files are logically placed when they are stored in this Share"
+ },
+ "AccessMode": {
+ "type": "array",
+ "description": "The type of access allowed for this share - AccessMode_t (undefined)",
+ "items": {
+ "type": "string"
+ }
+ },
+ "SharingID": {
+ "type": "string",
+ "description": "A local identifier common to the set of StorageShares which use the same underling extents"
+ },
+ "AccessLatency": {
+ "type": "string",
+ "description": "The maximum latency category under normal operating conditions",
+ "enum": ["nearline","offline","online"]
+ },
+ "RetentionPolicy": {
+ "type": "string",
+ "description": "The quality of data retention - RetentionPolicy_t"
+ },
+ "ExpirationMode": {
+ "type": "array",
+ "description": "Supported file lifetime modes",
+ "items": {
+ "type": "string",
+ "enum": ["neverexpire","releasewhenexpired","warnwhenexpired"]
+ }
+ },
+ "DefaultLifeTime": {
+ "type": "integer",
+ "description": "The default lifetime assigned to a new file"
+ },
+ "MaximumLifeTime": {
+ "type": "integer",
+ "description": "The maximum lifetime that can be requested for a file"
+ },
+ "Tag": {
+ "type": "string",
+ "description": "An identifier defined by a UserDomain"
+ },
+ "StorageShareCapacityID": {
+ "type": "array",
+ "description": "ID of the StorageShareCapacities associated with this share",
+ "items": {
+ "type": "string"
+ }
+ }
+ },
+ "required": ["ServingState","SharingID","AccessLatency"]
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageShareCapacity.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageShareCapacity.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageShareCapacity.json
new file mode 100644
index 0000000..f392c94
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/StorageShareCapacity.json
@@ -0,0 +1,33 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/StorageShareCapacity.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
+ "properties": {
+ "Type": {
+ "type": "string",
+ "description": "The type of storage capacity - StorageCapacity_t"
+ },
+ "TotalSize": {
+ "type": "integer",
+ "description": "The total amount of storage (GB)"
+ },
+ "FreeSize": {
+ "type": "integer",
+ "description": "The amount of available storage (GB)"
+ },
+ "UsedSize": {
+ "type": "integer",
+ "description": "The amount of used storage (GB)"
+ },
+ "ReservedSize": {
+ "type": "integer",
+ "description": "The amount storage which is not occupied, but has been reserved for use (GB)"
+ },
+ "StorageShareID": {
+ "type": "string",
+ "description": "The ID of the StorageShare related to this capacity"
+ }
+ },
+ "required": ["Type","StorageShareID"]
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ToComputingService.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ToComputingService.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ToComputingService.json
new file mode 100644
index 0000000..6d81b80
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ToComputingService.json
@@ -0,0 +1,32 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ToComputingService.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
+ "properties": {
+ "NetworkInfo": {
+ "type": "string",
+ "description": "Type of network connections between the Storage and Computing services (NetworkInfo_t)"
+ },
+ "Bandwidth": {
+ "type": "integer",
+ "description": "The normal bandwidth available between the Storage and Computing services (Mb/s)"
+ },
+ "StorageAccessProtocolID": {
+ "type": "array",
+ "description": "IDs of the protocols that can be used to access the StorageService",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ComputingServiceID": {
+ "type": "string",
+ "description": "The ID of the ComputingService"
+ },
+ "StorageServiceID": {
+ "type": "string",
+ "description": "The ID of the StorageService"
+ }
+ },
+ "required": ["ComputingServiceID","StorageServiceID"]
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ToStorageService.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ToStorageService.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ToStorageService.json
new file mode 100644
index 0000000..644f3d1
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/ToStorageService.json
@@ -0,0 +1,25 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ToStorageService.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
+ "properties": {
+ "LocalPath": {
+ "type": "string",
+ "description": "The path within the ComputingService that is used to access the StorageService"
+ },
+ "RemotePath": {
+ "type": "string",
+ "description": "The path in the StorageService which is associated with the LocalPath"
+ },
+ "ComputingServiceID": {
+ "type": "string",
+ "description": "The ID of the ComputingService"
+ },
+ "StorageServiceID": {
+ "type": "string",
+ "description": "The ID of the StorageService"
+ }
+ },
+ "required": ["LocalPath","RemotePath","ComputingServiceID","StorageServiceID"]
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/UserDomain.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/UserDomain.json b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/UserDomain.json
new file mode 100644
index 0000000..7acda31
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/schema/UserDomain.json
@@ -0,0 +1,58 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/UserDomain.json",
+ "type": "object",
+ "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Domain.json"}],
+ "properties": {
+ "Level": {
+ "type": "integer",
+ "description": "the number of hops to reach the root of the hierarchy of UserDomains"
+ },
+ "UserManagerID": {
+ "type": "array",
+ "description": "ID for the Endpoint of a Service managing users in this UserDomain",
+ "items": {
+ "type": "string"
+ }
+ },
+ "Member": {
+ "type": "array",
+ "description": "Identifiers for users in this UserDomain",
+ "items": {
+ "type": "string"
+ }
+ },
+ "PolicyID": {
+ "type": "array",
+ "description": "IDs for Policies associated with this UserDomain",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ChildDomainID": {
+ "type": "array",
+ "description": "IDs of UserDomains aggregated by this UserDomain",
+ "items": {
+ "type": "string"
+ }
+ },
+ "ParentDomainID": {
+ "type": "string",
+ "description": "The ID of the UserDomain that this UserDomain participates in"
+ },
+ "AccessPolicyID": {
+ "type": "array",
+ "description": "IDs of AccessPolicies associated with this UserDomain",
+ "items": {
+ "type": "string"
+ }
+ },
+ "MappingPolicyID": {
+ "type": "array",
+ "description": "IDs of MappingPolicies associated with this UserDomain",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/service.properties
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/service.properties b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/service.properties
new file mode 100644
index 0000000..391bfea
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/resources/service.properties
@@ -0,0 +1,58 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+
+#
+# Class which implemented Scheduler interface. It will be used to determine a Provider
+#
+scheduler.class= org.apache.airavata.core.gfac.scheduler.impl.SchedulerImpl
+
+#
+# Data Service Plugins classes
+#
+datachain.classes= org.apache.airavata.core.gfac.extension.data.RegistryDataService
+
+#
+# Pre execution Plugins classes. For example, GridFTP Input Staging
+#
+prechain.classes= org.apache.airavata.core.gfac.extension.pre.GridFtpInputStaging
+prechain.classes= org.apache.airavata.core.gfac.extension.pre.HttpInputStaging
+
+#
+# Post execution Plugins classes. For example, GridFTP Output Staging
+#
+postchain.classes= org.apache.airavata.core.gfac.extension.post.GridFtpOutputStaging
+postchain.classes= org.apache.airavata.core.gfac.extension.post.OutputRegister
+
+#
+# SSH private key location. It will be used by SSHProvider
+#
+# ssh.key=/home/user/.ssh/id_rsa
+# ssh.keypass=
+# ssh.username=usernameAtHost
+
+#
+# MyProxy credential. It will be used by GridFTP Plugins and GramProvider.
+#
+# myproxy.server=myproxy.teragrid.org
+# myproxy.user=username
+# myproxy.pass=password
+# myproxy.life=3600
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/test/java/org/apache/airavata/job/AMQPMonitorTest.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/test/java/org/apache/airavata/job/AMQPMonitorTest.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/test/java/org/apache/airavata/job/AMQPMonitorTest.java
new file mode 100644
index 0000000..daa2e91
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/test/java/org/apache/airavata/job/AMQPMonitorTest.java
@@ -0,0 +1,207 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.job;
+
+import com.google.common.eventbus.EventBus;
+import com.google.common.eventbus.Subscribe;
+import org.airavata.appcatalog.cpi.AppCatalog;
+import org.apache.aiaravata.application.catalog.data.impl.AppCatalogFactory;
+import org.apache.airavata.common.utils.MonitorPublisher;
+import org.apache.airavata.gfac.core.monitor.MonitorID;
+import org.apache.airavata.gfac.monitor.impl.push.amqp.AMQPMonitor;
+import org.apache.airavata.gsi.ssh.api.Cluster;
+import org.apache.airavata.gsi.ssh.api.SSHApiException;
+import org.apache.airavata.gsi.ssh.api.ServerInfo;
+import org.apache.airavata.gsi.ssh.api.authentication.GSIAuthenticationInfo;
+import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
+import org.apache.airavata.gsi.ssh.impl.PBSCluster;
+import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo;
+import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
+import org.apache.airavata.model.appcatalog.computeresource.DataMovementInterface;
+import org.apache.airavata.model.appcatalog.computeresource.DataMovementProtocol;
+import org.apache.airavata.model.appcatalog.computeresource.JobManagerCommand;
+import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionInterface;
+import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
+import org.apache.airavata.model.appcatalog.computeresource.ResourceJobManager;
+import org.apache.airavata.model.appcatalog.computeresource.ResourceJobManagerType;
+import org.apache.airavata.model.appcatalog.computeresource.SSHJobSubmission;
+import org.apache.airavata.model.appcatalog.computeresource.SecurityProtocol;
+import org.apache.airavata.model.messaging.event.JobStatusChangeEvent;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+public class AMQPMonitorTest {
+
+ private String myProxyUserName;
+ private String myProxyPassword;
+ private String certificateLocation;
+ private String pbsFilePath;
+ private String workingDirectory;
+ private MonitorPublisher monitorPublisher;
+ private BlockingQueue<MonitorID> finishQueue;
+ private BlockingQueue<MonitorID> pushQueue;
+ private Thread pushThread;
+ private String proxyFilePath;
+ private ComputeResourceDescription computeResourceDescription;
+ private final static Logger logger = LoggerFactory.getLogger(AMQPMonitorTest.class);
+
+ @Before
+ public void setUp() throws Exception {
+ System.setProperty("myproxy.username", "ogce");
+ System.setProperty("myproxy.password", "");
+ System.setProperty("basedir", "/Users/lahirugunathilake/work/airavata/sandbox/gsissh");
+ System.setProperty("gsi.working.directory", "/home1/01437/ogce");
+ System.setProperty("trusted.cert.location", "/Users/lahirugunathilake/Downloads/certificates");
+ System.setProperty("proxy.file.path", "/Users/lahirugunathilake/Downloads/x509up_u503876");
+ myProxyUserName = System.getProperty("myproxy.username");
+ myProxyPassword = System.getProperty("myproxy.password");
+ workingDirectory = System.getProperty("gsi.working.directory");
+ certificateLocation = System.getProperty("trusted.cert.location");
+ proxyFilePath = System.getProperty("proxy.file.path");
+ System.setProperty("connection.name", "xsede");
+ if (myProxyUserName == null || myProxyPassword == null || workingDirectory == null) {
+ System.out.println(">>>>>> Please run tests with my proxy user name and password. " +
+ "E.g :- mvn clean install -Dmyproxy.user=xxx -Dmyproxy.password=xxx -Dgsi.working.directory=/path<<<<<<<");
+ throw new Exception("Need my proxy user name password to run tests.");
+ }
+
+ monitorPublisher = new MonitorPublisher(new EventBus());
+ pushQueue = new LinkedBlockingQueue<MonitorID>();
+ finishQueue = new LinkedBlockingQueue<MonitorID>();
+
+
+ final AMQPMonitor amqpMonitor = new
+ AMQPMonitor(monitorPublisher,
+ pushQueue, finishQueue,proxyFilePath,"xsede",
+ Arrays.asList("info1.dyn.teragrid.org,info2.dyn.teragrid.org".split(",")));
+ try {
+ (new Thread(){
+ public void run(){
+ amqpMonitor.run();
+ }
+ }).start();
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ }
+ computeResourceDescription = new ComputeResourceDescription("TestComputerResoruceId", "TestHostName");
+ computeResourceDescription.setHostName("stampede-host");
+ computeResourceDescription.addToIpAddresses("login1.stampede.tacc.utexas.edu");
+ ResourceJobManager resourceJobManager = new ResourceJobManager("1234", ResourceJobManagerType.SLURM);
+ Map<JobManagerCommand, String> commandMap = new HashMap<JobManagerCommand, String>();
+ commandMap.put(JobManagerCommand.SUBMISSION, "test");
+ resourceJobManager.setJobManagerCommands(commandMap);
+ resourceJobManager.setJobManagerBinPath("/usr/bin/");
+ resourceJobManager.setPushMonitoringEndpoint("push"); // TODO - add monitor mode
+ SSHJobSubmission sshJobSubmission = new SSHJobSubmission("TestSSHJobSubmissionInterfaceId", SecurityProtocol.GSI,
+ resourceJobManager);
+
+ AppCatalog appCatalog = AppCatalogFactory.getAppCatalog();
+ String jobSubmissionID = appCatalog.getComputeResource().addSSHJobSubmission(sshJobSubmission);
+
+ JobSubmissionInterface jobSubmissionInterface = new JobSubmissionInterface(jobSubmissionID, JobSubmissionProtocol.SSH, 1);
+
+ computeResourceDescription.addToJobSubmissionInterfaces(jobSubmissionInterface);
+ computeResourceDescription.addToDataMovementInterfaces(new DataMovementInterface("4532", DataMovementProtocol.SCP, 1));
+
+ }
+
+ @Test
+ public void testAMQPMonitor() throws SSHApiException {
+ /* now have to submit a job to some machine and add that job to the queue */
+ //Create authentication
+ GSIAuthenticationInfo authenticationInfo
+ = new MyProxyAuthenticationInfo(myProxyUserName, myProxyPassword, "myproxy.teragrid.org",
+ 7512, 17280000, certificateLocation);
+
+ // Server info
+ ServerInfo serverInfo = new ServerInfo("ogce", "login1.stampede.tacc.utexas.edu",2222);
+
+
+ Cluster pbsCluster = new
+ PBSCluster(serverInfo, authenticationInfo, org.apache.airavata.gsi.ssh.util.CommonUtils.getPBSJobManager("/usr/bin/"));
+
+
+ // Execute command
+ System.out.println("Target PBS file path: " + workingDirectory);
+ // constructing the job object
+ String jobName = "GSI_SSH_SLEEP_JOB";
+ JobDescriptor jobDescriptor = new JobDescriptor();
+ jobDescriptor.setWorkingDirectory(workingDirectory);
+ jobDescriptor.setShellName("/bin/bash");
+ jobDescriptor.setJobName(jobName);
+ jobDescriptor.setExecutablePath("/bin/echo");
+ jobDescriptor.setAllEnvExport(true);
+ jobDescriptor.setMailOptions("n");
+ jobDescriptor.setStandardOutFile(workingDirectory + File.separator + "application.out");
+ jobDescriptor.setStandardErrorFile(workingDirectory + File.separator + "application.err");
+ jobDescriptor.setNodes(1);
+ jobDescriptor.setProcessesPerNode(1);
+ jobDescriptor.setQueueName("normal");
+ jobDescriptor.setMaxWallTime("60");
+ jobDescriptor.setAcountString("TG-STA110014S");
+ List<String> inputs = new ArrayList<String>();
+ jobDescriptor.setOwner("ogce");
+ inputs.add("Hello World");
+ jobDescriptor.setInputValues(inputs);
+ //finished construction of job object
+ System.out.println(jobDescriptor.toXML());
+ String jobID = pbsCluster.submitBatchJob(jobDescriptor);
+ System.out.println(jobID);
+ try {
+ pushQueue.add(new MonitorID(computeResourceDescription, jobID,null,null,null, "ogce", jobName));
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ try {
+ pushThread.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+ class InnerClassAMQP{
+ @Subscribe
+ private void getStatus(JobStatusChangeEvent status){
+ Assert.assertNotNull(status);
+ pushThread.interrupt();
+ }
+ }
+ monitorPublisher.registerListener(new InnerClassAMQP());
+// try {
+// pushThread.join(5000);
+// Iterator<MonitorID> iterator = pushQueue.iterator();
+// MonitorID next = iterator.next();
+// org.junit.Assert.assertNotNull(next.getStatus());
+// } catch (Exception e) {
+// e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+// }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/test/java/org/apache/airavata/job/QstatMonitorTestWithMyProxyAuth.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/test/java/org/apache/airavata/job/QstatMonitorTestWithMyProxyAuth.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/test/java/org/apache/airavata/job/QstatMonitorTestWithMyProxyAuth.java
new file mode 100644
index 0000000..610934e
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/test/java/org/apache/airavata/job/QstatMonitorTestWithMyProxyAuth.java
@@ -0,0 +1,172 @@
+///*
+// *
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements. See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership. The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing,
+// * software distributed under the License is distributed on an
+// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// * KIND, either express or implied. See the License for the
+// * specific language governing permissions and limitations
+// * under the License.
+// *
+//*/
+//package org.apache.airavata.job;
+//
+//import java.io.File;
+//import java.util.ArrayList;
+//import java.util.List;
+//import java.util.concurrent.BlockingQueue;
+//import java.util.concurrent.LinkedBlockingQueue;
+//
+//import org.apache.airavata.common.utils.MonitorPublisher;
+//import org.apache.airavata.commons.gfac.type.HostDescription;
+//import org.apache.airavata.gfac.core.monitor.MonitorID;
+//import org.apache.airavata.gfac.monitor.HPCMonitorID;
+//import org.apache.airavata.gfac.monitor.UserMonitorData;
+//import org.apache.airavata.gfac.monitor.impl.pull.qstat.HPCPullMonitor;
+//import org.apache.airavata.gsi.ssh.api.Cluster;
+//import org.apache.airavata.gsi.ssh.api.SSHApiException;
+//import org.apache.airavata.gsi.ssh.api.ServerInfo;
+//import org.apache.airavata.gsi.ssh.api.authentication.GSIAuthenticationInfo;
+//import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
+//import org.apache.airavata.gsi.ssh.impl.PBSCluster;
+//import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo;
+//import org.apache.airavata.gsi.ssh.util.CommonUtils;
+//import org.apache.airavata.model.messaging.event.JobStatusChangeEvent;
+//import org.apache.airavata.schemas.gfac.GsisshHostType;
+//import org.junit.Assert;
+//import org.testng.annotations.Test;
+//
+//import com.google.common.eventbus.EventBus;
+//import com.google.common.eventbus.Subscribe;
+//
+//public class QstatMonitorTestWithMyProxyAuth {
+// private String myProxyUserName;
+// private String myProxyPassword;
+// private String certificateLocation;
+// private String pbsFilePath;
+// private String workingDirectory;
+// private HostDescription hostDescription;
+// private MonitorPublisher monitorPublisher;
+// private BlockingQueue<UserMonitorData> pullQueue;
+// private Thread monitorThread;
+//
+// @org.testng.annotations.BeforeClass
+// public void setUp() throws Exception {
+//// System.setProperty("myproxy.username", "ogce");
+//// System.setProperty("myproxy.password", "");
+//// System.setProperty("basedir", "/Users/lahirugunathilake/work/airavata/sandbox/gsissh");
+//// System.setProperty("gsi.working.directory", "/home/ogce");
+//// System.setProperty("trusted.cert.location", "/Users/lahirugunathilake/Downloads/certificates");
+// myProxyUserName = System.getProperty("myproxy.username");
+// myProxyPassword = System.getProperty("myproxy.password");
+// workingDirectory = System.getProperty("gsi.working.directory");
+// certificateLocation = System.getProperty("trusted.cert.location");
+// if (myProxyUserName == null || myProxyPassword == null || workingDirectory == null) {
+// System.out.println(">>>>>> Please run tests with my proxy user name and password. " +
+// "E.g :- mvn clean install -Dmyproxy.username=xxx -Dmyproxy.password=xxx -Dgsi.working.directory=/path<<<<<<<");
+// throw new Exception("Need my proxy user name password to run tests.");
+// }
+//
+// monitorPublisher = new MonitorPublisher(new EventBus());
+// class InnerClassQstat {
+//
+// @Subscribe
+// private void getStatus(JobStatusChangeEvent status) {
+// Assert.assertNotNull(status);
+// System.out.println(status.getState().toString());
+// monitorThread.interrupt();
+// }
+// }
+// monitorPublisher.registerListener(this);
+// pullQueue = new LinkedBlockingQueue<UserMonitorData>();
+// final HPCPullMonitor qstatMonitor = new
+// HPCPullMonitor(pullQueue, monitorPublisher);
+// try {
+// (new Thread(){
+// public void run(){
+// qstatMonitor.run();
+// }
+// }).start();
+// } catch (Exception e) {
+// e.printStackTrace();
+// }
+//
+// hostDescription = new HostDescription(GsisshHostType.type);
+// hostDescription.getType().setHostAddress("trestles.sdsc.edu");
+// hostDescription.getType().setHostName("gsissh-gordon");
+// ((GsisshHostType) hostDescription.getType()).setPort(22);
+// ((GsisshHostType)hostDescription.getType()).setInstalledPath("/opt/torque/bin/");
+// }
+//
+// @Test
+// public void testQstatMonitor() throws SSHApiException {
+// /* now have to submit a job to some machine and add that job to the queue */
+// //Create authentication
+// GSIAuthenticationInfo authenticationInfo
+// = new MyProxyAuthenticationInfo(myProxyUserName, myProxyPassword, "myproxy.teragrid.org",
+// 7512, 17280000, certificateLocation);
+//
+// // Server info
+// ServerInfo serverInfo = new ServerInfo("ogce", hostDescription.getType().getHostAddress());
+//
+//
+// Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, CommonUtils.getPBSJobManager("/opt/torque/bin/"));
+//
+//
+// // Execute command
+// System.out.println("Target PBS file path: " + workingDirectory);
+// // constructing the job object
+// JobDescriptor jobDescriptor = new JobDescriptor();
+// jobDescriptor.setWorkingDirectory(workingDirectory);
+// jobDescriptor.setShellName("/bin/bash");
+// jobDescriptor.setJobName("GSI_SSH_SLEEP_JOB");
+// jobDescriptor.setExecutablePath("/bin/echo");
+// jobDescriptor.setAllEnvExport(true);
+// jobDescriptor.setMailOptions("n");
+// jobDescriptor.setStandardOutFile(workingDirectory + File.separator + "application.out");
+// jobDescriptor.setStandardErrorFile(workingDirectory + File.separator + "application.err");
+// jobDescriptor.setNodes(1);
+// jobDescriptor.setProcessesPerNode(1);
+// jobDescriptor.setQueueName("normal");
+// jobDescriptor.setMaxWallTime("60");
+// jobDescriptor.setAcountString("sds128");
+// List<String> inputs = new ArrayList<String>();
+// jobDescriptor.setOwner("ogce");
+// inputs.add("Hello World");
+// jobDescriptor.setInputValues(inputs);
+// //finished construction of job object
+// System.out.println(jobDescriptor.toXML());
+// for (int i = 0; i < 1; i++) {
+// String jobID = pbsCluster.submitBatchJob(jobDescriptor);
+// System.out.println("Job submitted successfully, Job ID: " + jobID);
+// MonitorID monitorID = new HPCMonitorID(hostDescription, jobID,null,null,null, "ogce","");
+// ((HPCMonitorID)monitorID).setAuthenticationInfo(authenticationInfo);
+// try {
+// org.apache.airavata.gfac.monitor.util.CommonUtils.addMonitortoQueue(pullQueue, monitorID, jobExecutionContext);
+// } catch (Exception e) {
+// e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+// }
+// }
+// try {
+//
+// monitorThread.join();
+// } catch (Exception e) {
+// e.printStackTrace();
+// }
+// }
+//
+// @Subscribe
+// public void testCaseShutDown(JobStatusChangeEvent status) {
+// Assert.assertNotNull(status.getState());
+// monitorThread.stop();
+// }
+//}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/test/resources/PBSTemplate.xslt
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/test/resources/PBSTemplate.xslt b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/test/resources/PBSTemplate.xslt
new file mode 100644
index 0000000..e749e9c
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/test/resources/PBSTemplate.xslt
@@ -0,0 +1,73 @@
+<!--Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to you under
+ the Apache License, Version 2.0 (theÏ "License"); you may not use this file except in compliance with the License. You may
+ obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to
+ in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+ ANY ~ KIND, either express or implied. See the License for the specific language governing permissions and limitations under
+ the License. -->
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:ns="http://airavata.apache.org/gsi/ssh/2012/12">
+<xsl:output method="text" />
+<xsl:template match="/ns:JobDescriptor">
+#! /bin/sh
+# PBS batch job script built by Globus job manager
+# <xsl:choose>
+ <xsl:when test="ns:shellName">
+##PBS -S <xsl:value-of select="ns:shellName"/>
+ </xsl:when></xsl:choose>
+ <xsl:choose>
+ <xsl:when test="ns:queueName">
+#PBS -q <xsl:value-of select="ns:queueName"/>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:choose>
+ <xsl:when test="ns:mailOptions">
+#PBS -m <xsl:value-of select="ns:mailOptions"/>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:choose>
+<xsl:when test="ns:acountString">
+#PBS -A <xsl:value-of select="ns:acountString"/>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:choose>
+ <xsl:when test="ns:maxWallTime">
+#PBS -l walltime=<xsl:value-of select="ns:maxWallTime"/>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:choose>
+ <xsl:when test="ns:standardOutFile">
+#PBS -o <xsl:value-of select="ns:standardOutFile"/>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:choose>
+ <xsl:when test="ns:standardOutFile">
+#PBS -e <xsl:value-of select="ns:standardErrorFile"/>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:choose>
+ <xsl:when test="(ns:nodes) and (ns:processesPerNode)">
+#PBS -l nodes=<xsl:value-of select="ns:nodes"/>:ppn=<xsl:value-of select="ns:processesPerNode"/>
+<xsl:text>
</xsl:text>
+ </xsl:when>
+ </xsl:choose>
+<xsl:for-each select="ns:exports/ns:name">
+<xsl:value-of select="."/>=<xsl:value-of select="./@value"/><xsl:text>
</xsl:text>
+export<xsl:text> </xsl:text><xsl:value-of select="."/>
+<xsl:text>
</xsl:text>
+</xsl:for-each>
+<xsl:for-each select="ns:preJobCommands/ns:command">
+ <xsl:value-of select="."/><xsl:text> </xsl:text>
+ </xsl:for-each>
+cd <xsl:text> </xsl:text><xsl:value-of select="ns:workingDirectory"/><xsl:text>
</xsl:text>
+ <xsl:choose><xsl:when test="ns:jobSubmitterCommand">
+<xsl:value-of select="ns:jobSubmitterCommand"/><xsl:text> </xsl:text></xsl:when></xsl:choose><xsl:value-of select="ns:executablePath"/><xsl:text> </xsl:text>
+<xsl:for-each select="ns:inputs/ns:input">
+ <xsl:value-of select="."/><xsl:text> </xsl:text>
+ </xsl:for-each>
+<xsl:for-each select="ns:postJobCommands/ns:command">
+ <xsl:value-of select="."/><xsl:text> </xsl:text>
+</xsl:for-each>
+
+</xsl:template>
+
+</xsl:stylesheet>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/test/resources/echo.bat
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/test/resources/echo.bat b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/test/resources/echo.bat
new file mode 100644
index 0000000..c6b849b
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/test/resources/echo.bat
@@ -0,0 +1,22 @@
+::
+::
+:: Licensed to the Apache Software Foundation (ASF) under one
+:: or more contributor license agreements. See the NOTICE file
+:: distributed with this work for additional information
+:: regarding copyright ownership. The ASF licenses this file
+:: to you under the Apache License, Version 2.0 (the
+:: "License"); you may not use this file except in compliance
+:: with the License. You may obtain a copy of the License at
+::
+:: http://www.apache.org/licenses/LICENSE-2.0
+::
+:: Unless required by applicable law or agreed to in writing,
+:: software distributed under the License is distributed on an
+:: "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+:: KIND, either express or implied. See the License for the
+:: specific language governing permissions and limitations
+:: under the License.
+::
+::
+@echo off
+echo %1^=%2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/test/resources/logging.properties
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/test/resources/logging.properties b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/test/resources/logging.properties
new file mode 100644
index 0000000..0584d38
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/test/resources/logging.properties
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+#default/fallback log4j configuration
+#
+
+# Set root logger level to WARN and its only appender to A1.
+log4j.rootLogger=INFO, A1, A2
+
+# A1 is set to be a rolling file appender with default params
+log4j.appender.A1=org.apache.log4j.RollingFileAppender
+log4j.appender.A1.File=target/seclogs.txt
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %c %x - %m%n
+
+# A2 is a console appender
+log4j.appender.A2=org.apache.log4j.ConsoleAppender
+
+# A2 uses PatternLayout.
+log4j.appender.A2.layout=org.apache.log4j.PatternLayout
+log4j.appender.A2.layout.ConversionPattern=%d [%t] %-5p %c{1} %x - %m%n
+
+log4j.logger.unicore.security=INFO
+
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/pom.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/pom.xml b/modules/gfac/gfac-monitor/pom.xml
index c4a6e2f..66c5875 100644
--- a/modules/gfac/gfac-monitor/pom.xml
+++ b/modules/gfac/gfac-monitor/pom.xml
@@ -17,157 +17,13 @@
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>airavata-gfac-hpc-monitor</artifactId>
- <name>Airavata GFac Grid Job Monitor</name>
- <description>The Grid related monitoring implementation</description>
+ <artifactId>airavata-gfac-monitor</artifactId>
+ <packaging>pom</packaging>
+ <name>Airavata GFac Monitor</name>
<url>http://airavata.apache.org/</url>
-
- <dependencies>
- <!-- Logging -->
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
-
- <!-- GFAC schemas -->
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>airavata-gfac-core</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>airavata-gfac-gsissh</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>airavata-gfac-ssh</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>airavata-workflow-execution-context</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>airavata-registry-cpi</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>airavata-jpa-registry</artifactId>
- <version>${project.version}</version>
- </dependency>
- <!-- Workflow Tracking -->
- <!--<dependency>-->
- <!--<groupId>org.apache.airavata</groupId>-->
- <!--<artifactId>airavata-workflow-tracking</artifactId>-->
- <!--<version>${project.version}</version>-->
- <!--</dependency>-->
- <!-- Credential Store -->
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>airavata-credential-store</artifactId>
- <version>${project.version}</version>
- </dependency>
- <!-- Test -->
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.testng</groupId>
- <artifactId>testng</artifactId>
- <version>6.1.1</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>airavata-server-configuration</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>airavata-client-configuration</artifactId>
- <scope>test</scope>
- </dependency>
-
- <!-- Guava -->
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>12.0</version>
- </dependency>
- <!-- gsi-ssh api dependencies -->
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>gsissh</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>airavata-data-models</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>com.jcraft</groupId>
- <artifactId>jsch</artifactId>
- <version>0.1.50</version>
- </dependency>
- <dependency>
- <groupId>org.apache.xmlbeans</groupId>
- <artifactId>xmlbeans</artifactId>
- <version>${xmlbeans.version}</version>
- </dependency>
- <!-- this is the dependency for amqp implementation -->
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>2.0.0</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <skip>false</skip>
- <forkMode>always</forkMode>
- <failIfNoTests>false</failIfNoTests>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.jsonschema2pojo</groupId>
- <artifactId>jsonschema2pojo-maven-plugin</artifactId>
- <version>0.4.0</version>
- <configuration>
- <sourceDirectory>${basedir}/src/main/resources/schema</sourceDirectory>
- <targetPackage>org.apache.airavata</targetPackage>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>generate</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
+ <modules>
+ <module>gfac-hpc-monitor</module>
+ <module>gfac-email-monitor</module>
+ </modules>
</project>
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/HPCMonitorID.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/HPCMonitorID.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/HPCMonitorID.java
deleted file mode 100644
index ae463a7..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/HPCMonitorID.java
+++ /dev/null
@@ -1,107 +0,0 @@
-package org.apache.airavata.gfac.monitor;/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.SecurityContext;
-import org.apache.airavata.gfac.core.context.JobExecutionContext;
-import org.apache.airavata.gfac.core.monitor.MonitorID;
-import org.apache.airavata.gfac.gsissh.security.GSISecurityContext;
-import org.apache.airavata.gfac.ssh.security.SSHSecurityContext;
-import org.apache.airavata.gsi.ssh.api.ServerInfo;
-import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
-import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo;
-import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.sql.Timestamp;
-import java.util.Date;
-
-public class HPCMonitorID extends MonitorID {
- private final static Logger logger = LoggerFactory.getLogger(HPCMonitorID.class);
-
-
- private AuthenticationInfo authenticationInfo = null;
-
- public HPCMonitorID(ComputeResourceDescription computeResourceDescription, String jobID, String taskID, String workflowNodeID,
- String experimentID, String userName,String jobName) {
- super(computeResourceDescription, jobID, taskID, workflowNodeID, experimentID, userName,jobName);
- setComputeResourceDescription(computeResourceDescription);
- setJobStartedTime(new Timestamp((new Date()).getTime()));
- setUserName(userName);
- setJobID(jobID);
- setTaskID(taskID);
- setExperimentID(experimentID);
- setWorkflowNodeID(workflowNodeID);
- }
-
- public HPCMonitorID(AuthenticationInfo authenticationInfo, JobExecutionContext jobExecutionContext) {
- super(jobExecutionContext);
- this.authenticationInfo = authenticationInfo;
- if (this.authenticationInfo != null) {
- try {
- String hostAddress = jobExecutionContext.getHostName();
- SecurityContext securityContext = jobExecutionContext.getSecurityContext(hostAddress);
- ServerInfo serverInfo = null;
- if (securityContext != null) {
- if (securityContext instanceof GSISecurityContext){
- serverInfo = (((GSISecurityContext) securityContext).getPbsCluster()).getServerInfo();
- if (serverInfo.getUserName() != null) {
- setUserName(serverInfo.getUserName());
- }
- }
- if (securityContext instanceof SSHSecurityContext){
- serverInfo = (((SSHSecurityContext) securityContext).getPbsCluster()).getServerInfo();
- if (serverInfo.getUserName() != null) {
- setUserName(serverInfo.getUserName());
- }
- }
- }
- } catch (GFacException e) {
- logger.error("Error while getting security context", e);
- }
- }
- }
-
- public HPCMonitorID(ComputeResourceDescription computeResourceDescription, String jobID, String taskID, String workflowNodeID, String experimentID, String userName, AuthenticationInfo authenticationInfo) {
- setComputeResourceDescription(computeResourceDescription);
- setJobStartedTime(new Timestamp((new Date()).getTime()));
- this.authenticationInfo = authenticationInfo;
- // if we give myproxyauthenticationInfo, so we try to use myproxy user as the user
- if (this.authenticationInfo != null) {
- if (this.authenticationInfo instanceof MyProxyAuthenticationInfo) {
- setUserName(((MyProxyAuthenticationInfo) this.authenticationInfo).getUserName());
- }
- }
- setJobID(jobID);
- setTaskID(taskID);
- setExperimentID(experimentID);
- setWorkflowNodeID(workflowNodeID);
- }
-
- public AuthenticationInfo getAuthenticationInfo() {
- return authenticationInfo;
- }
-
- public void setAuthenticationInfo(AuthenticationInfo authenticationInfo) {
- this.authenticationInfo = authenticationInfo;
- }
-}
[2/7] airavata git commit: Add new gfac email monitor module to gfac
monitor module and Intorduced gfac-hpc-monitor module to keep all hoc-monitor
code
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/JSONMessageParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/JSONMessageParser.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/JSONMessageParser.java
deleted file mode 100644
index 72c77d5..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/JSONMessageParser.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.monitor.impl.push.amqp;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.airavata.ComputingActivity;
-import org.apache.airavata.gfac.monitor.core.MessageParser;
-import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
-import org.apache.airavata.model.workspace.experiment.JobState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-
-public class JSONMessageParser implements MessageParser {
- private final static Logger logger = LoggerFactory.getLogger(JSONMessageParser.class);
-
- public JobState parseMessage(String message)throws AiravataMonitorException {
- /*todo write a json message parser here*/
- logger.debug(message);
- ObjectMapper objectMapper = new ObjectMapper();
- try {
- ComputingActivity computingActivity = objectMapper.readValue(message.getBytes(), ComputingActivity.class);
- logger.info(computingActivity.getIDFromEndpoint());
- List<String> stateList = computingActivity.getState();
- JobState jobState = null;
- for (String aState : stateList) {
- jobState = getStatusFromString(aState);
- }
- // we get the last value of the state array
- return jobState;
- } catch (IOException e) {
- throw new AiravataMonitorException(e);
- }
- }
-
-private JobState getStatusFromString(String status) {
- logger.info("parsing the job status returned : " + status);
- if(status != null){
- if("ipf:finished".equals(status)){
- return JobState.COMPLETE;
- }else if("ipf:pending".equals(status)|| "ipf:starting".equals(status)){
- return JobState.QUEUED;
- }else if("ipf:running".equals(status) || "ipf:finishing".equals(status)){
- return JobState.ACTIVE;
- }else if ("ipf:held".equals(status) || "ipf:teminating".equals(status) || "ipf:teminated".equals(status)) {
- return JobState.HELD;
- } else if ("ipf:suspending".equals(status)) {
- return JobState.SUSPENDED;
- }else if ("ipf:failed".equals(status)) {
- return JobState.FAILED;
- }else if ("ipf:unknown".equals(status)){
- return JobState.UNKNOWN;
- }
- }
- return JobState.UNKNOWN;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/SimpleJobFinishConsumer.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/SimpleJobFinishConsumer.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/SimpleJobFinishConsumer.java
deleted file mode 100644
index c4275f1..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/SimpleJobFinishConsumer.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.monitor.impl.push.amqp;
-
-import com.rabbitmq.client.Channel;
-import com.rabbitmq.client.Connection;
-import com.rabbitmq.client.ConnectionFactory;
-import com.rabbitmq.client.QueueingConsumer;
-import org.apache.airavata.common.utils.Constants;
-import org.apache.airavata.common.utils.ServerSettings;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-public class SimpleJobFinishConsumer {
- private final static Logger logger = LoggerFactory.getLogger(SimpleJobFinishConsumer.class);
-
- private List<String> completedJobsFromPush;
-
- public SimpleJobFinishConsumer(List<String> completedJobsFromPush) {
- this.completedJobsFromPush = completedJobsFromPush;
- }
-
- public void listen() {
- try {
- String queueName = ServerSettings.getSetting(Constants.GFAC_SERVER_PORT, "8950");
- String uri = "amqp://localhost";
-
- ConnectionFactory connFactory = new ConnectionFactory();
- connFactory.setUri(uri);
- Connection conn = connFactory.newConnection();
- logger.info("--------Created the connection to Rabbitmq server successfully-------");
-
- final Channel ch = conn.createChannel();
-
- logger.info("--------Created the channel with Rabbitmq server successfully-------");
-
- ch.queueDeclare(queueName, false, false, false, null);
-
- logger.info("--------Declare the queue " + queueName + " in Rabbitmq server successfully-------");
-
- final QueueingConsumer consumer = new QueueingConsumer(ch);
- ch.basicConsume(queueName, consumer);
- (new Thread() {
- public void run() {
- try {
- while (true) {
- QueueingConsumer.Delivery delivery = consumer.nextDelivery();
- String message = new String(delivery.getBody());
- logger.info("---------------- Job Finish message received:" + message + " --------------");
- synchronized (completedJobsFromPush) {
- completedJobsFromPush.add(message);
- }
- ch.basicAck(delivery.getEnvelope().getDeliveryTag(), false);
- }
- } catch (Exception ex) {
- logger.error("--------Cannot connect to a RabbitMQ Server--------" , ex);
- }
- }
-
- }).start();
- } catch (Exception ex) {
- logger.error("Cannot connect to a RabbitMQ Server: " , ex);
- logger.info("------------- Push monitoring for HPC jobs is disabled -------------");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/UnRegisterWorker.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/UnRegisterWorker.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/UnRegisterWorker.java
deleted file mode 100644
index a701326..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/UnRegisterWorker.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.monitor.impl.push.amqp;
-
-import com.google.common.eventbus.Subscribe;
-import com.rabbitmq.client.Channel;
-import org.apache.airavata.gfac.core.monitor.MonitorID;
-import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
-import org.apache.airavata.gfac.monitor.util.CommonUtils;
-import org.apache.airavata.model.messaging.event.JobStatusChangeEvent;
-import org.apache.airavata.model.workspace.experiment.JobState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Map;
-
-public class UnRegisterWorker{
- private final static Logger logger = LoggerFactory.getLogger(UnRegisterWorker.class);
- private Map<String, Channel> availableChannels;
-
- public UnRegisterWorker(Map<String, Channel> channels) {
- this.availableChannels = channels;
- }
-
- @Subscribe
- private boolean unRegisterListener(JobStatusChangeEvent jobStatus, MonitorID monitorID) throws AiravataMonitorException {
- String channelID = CommonUtils.getChannelID(monitorID);
- if (JobState.FAILED.equals(jobStatus.getState()) || JobState.COMPLETE.equals(jobStatus.getState())){
- Channel channel = availableChannels.get(channelID);
- if (channel == null) {
- logger.error("Already Unregistered the listener");
- throw new AiravataMonitorException("Already Unregistered the listener");
- } else {
- try {
- channel.queueUnbind(channel.queueDeclare().getQueue(), "glue2.computing_activity", CommonUtils.getRoutingKey(monitorID));
- channel.close();
- channel.getConnection().close();
- availableChannels.remove(channelID);
- } catch (IOException e) {
- logger.error("Error unregistering the listener");
- throw new AiravataMonitorException("Error unregistering the listener");
- }
- }
- }
- return true;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/AMQPConnectionUtil.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/AMQPConnectionUtil.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/AMQPConnectionUtil.java
deleted file mode 100644
index 6a4ed3b..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/AMQPConnectionUtil.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.monitor.util;
-
-import com.rabbitmq.client.Connection;
-import com.rabbitmq.client.ConnectionFactory;
-import com.rabbitmq.client.DefaultSaslConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.net.ssl.KeyManagerFactory;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManagerFactory;
-import java.security.KeyStore;
-import java.util.Collections;
-import java.util.List;
-
-public class AMQPConnectionUtil {
- private final static Logger logger = LoggerFactory.getLogger(AMQPConnectionUtil.class);
- public static Connection connect(List<String>hosts,String vhost, String proxyFile) {
- Collections.shuffle(hosts);
- for (String host : hosts) {
- Connection connection = connect(host, vhost, proxyFile);
- if (host != null) {
- System.out.println("connected to " + host);
- return connection;
- }
- }
- return null;
- }
-
- public static Connection connect(String host, String vhost, String proxyFile) {
- Connection connection;
- try {
- String keyPassPhrase = "test123";
- KeyStore ks = X509Helper.keyStoreFromPEM(proxyFile, keyPassPhrase);
- KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509");
- kmf.init(ks, keyPassPhrase.toCharArray());
-
- KeyStore tks = X509Helper.trustKeyStoreFromCertDir();
- TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509");
- tmf.init(tks);
-
- SSLContext c = SSLContext.getInstance("SSLv3");
- c.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null);
-
- ConnectionFactory factory = new ConnectionFactory();
- factory.setHost(host);
- factory.setPort(5671);
- factory.useSslProtocol(c);
- factory.setVirtualHost(vhost);
- factory.setSaslConfig(DefaultSaslConfig.EXTERNAL);
-
- connection = factory.newConnection();
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- return null;
- }
- return connection;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/CommonUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/CommonUtils.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/CommonUtils.java
deleted file mode 100644
index 15b7241..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/CommonUtils.java
+++ /dev/null
@@ -1,298 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.monitor.util;
-
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.logger.AiravataLogger;
-import org.apache.airavata.common.logger.AiravataLoggerFactory;
-import org.apache.airavata.common.utils.AiravataZKUtils;
-import org.apache.airavata.common.utils.Constants;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.core.context.JobExecutionContext;
-import org.apache.airavata.gfac.core.handler.GFacHandler;
-import org.apache.airavata.gfac.core.handler.GFacHandlerConfig;
-import org.apache.airavata.gfac.core.monitor.MonitorID;
-import org.apache.airavata.gfac.monitor.HostMonitorData;
-import org.apache.airavata.gfac.monitor.UserMonitorData;
-import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
-import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
-import org.apache.zookeeper.*;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.CountDownLatch;
-
-public class CommonUtils {
- private final static AiravataLogger logger = AiravataLoggerFactory.getLogger(CommonUtils.class);
-
- public static String getChannelID(MonitorID monitorID) {
- return monitorID.getUserName() + "-" + monitorID.getComputeResourceDescription().getHostName();
- }
-
- public static String getRoutingKey(MonitorID monitorID) {
- return "*." + monitorID.getUserName() + "." + monitorID.getComputeResourceDescription().getIpAddresses().get(0);
- }
-
- public static String getChannelID(String userName,String hostAddress) {
- return userName + "-" + hostAddress;
- }
-
- public static String getRoutingKey(String userName,String hostAddress) {
- return "*." + userName + "." + hostAddress;
- }
-
- public static void addMonitortoQueue(BlockingQueue<UserMonitorData> queue, MonitorID monitorID, JobExecutionContext jobExecutionContext) throws AiravataMonitorException {
- synchronized (queue) {
- Iterator<UserMonitorData> iterator = queue.iterator();
- while (iterator.hasNext()) {
- UserMonitorData next = iterator.next();
- if (next.getUserName().equals(monitorID.getUserName())) {
- // then this is the right place to update
- List<HostMonitorData> monitorIDs = next.getHostMonitorData();
- for (HostMonitorData host : monitorIDs) {
- if (isEqual(host.getComputeResourceDescription(), monitorID.getComputeResourceDescription())) {
- // ok we found right place to add this monitorID
- host.addMonitorIDForHost(monitorID);
- logger.debugId(monitorID.getJobID(), "Added new job to the monitoring queue, experiment {}," +
- " task {}", monitorID.getExperimentID(), monitorID.getTaskID());
- return;
- }
- }
- // there is a userMonitor object for this user name but no Hosts for this host
- // so we have to create new Hosts
- HostMonitorData hostMonitorData = new HostMonitorData(jobExecutionContext);
- hostMonitorData.addMonitorIDForHost(monitorID);
- next.addHostMonitorData(hostMonitorData);
- logger.debugId(monitorID.getJobID(), "Added new job to the monitoring queue, experiment {}," +
- " task {}", monitorID.getExperimentID(), monitorID.getTaskID());
- return;
- }
- }
- HostMonitorData hostMonitorData = new HostMonitorData(jobExecutionContext);
- hostMonitorData.addMonitorIDForHost(monitorID);
-
- UserMonitorData userMonitorData = new UserMonitorData(monitorID.getUserName());
- userMonitorData.addHostMonitorData(hostMonitorData);
- try {
- queue.put(userMonitorData);
- logger.debugId(monitorID.getJobID(), "Added new job to the monitoring queue, experiment {}," +
- " task {}", monitorID.getExperimentID(), monitorID.getTaskID());
- } catch (InterruptedException e) {
- throw new AiravataMonitorException(e);
- }
- }
- }
-
- private static boolean isEqual(ComputeResourceDescription comRes_1, ComputeResourceDescription comRes_2) {
- return comRes_1.getComputeResourceId().equals(comRes_2.getComputeResourceId()) &&
- comRes_1.getHostName().equals(comRes_2.getHostName());
- }
-
- public static boolean isTheLastJobInQueue(BlockingQueue<MonitorID> queue,MonitorID monitorID){
- Iterator<MonitorID> iterator = queue.iterator();
- while(iterator.hasNext()){
- MonitorID next = iterator.next();
- if (monitorID.getUserName().equals(next.getUserName()) &&
- CommonUtils.isEqual(monitorID.getComputeResourceDescription(), next.getComputeResourceDescription())) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * This method doesn't have to be synchronized because it will be invoked by HPCPullMonitor which already synchronized
- * @param monitorID
- * @throws AiravataMonitorException
- */
- public static void removeMonitorFromQueue(UserMonitorData userMonitorData, MonitorID monitorID) throws AiravataMonitorException {
- if (userMonitorData.getUserName().equals(monitorID.getUserName())) {
- // then this is the right place to update
- List<HostMonitorData> hostMonitorData = userMonitorData.getHostMonitorData();
- Iterator<HostMonitorData> iterator1 = hostMonitorData.iterator();
- while (iterator1.hasNext()) {
- HostMonitorData iHostMonitorID = iterator1.next();
- if (isEqual(iHostMonitorID.getComputeResourceDescription(), monitorID.getComputeResourceDescription())) {
- Iterator<MonitorID> iterator2 = iHostMonitorID.getMonitorIDs().iterator();
- while (iterator2.hasNext()) {
- MonitorID iMonitorID = iterator2.next();
- if (iMonitorID.getJobID().equals(monitorID.getJobID())
- || iMonitorID.getJobName().equals(monitorID.getJobName())) {
- // OK we found the object, we cannot do list.remove(object) states of two objects
- // could be different, thats why we check the jobID
- iterator2.remove();
- logger.infoId(monitorID.getJobID(), "Removed the jobId: {} JobName: {} from monitoring last " +
- "status:{}", monitorID.getJobID(),monitorID.getJobName(), monitorID.getStatus().toString());
-
- return;
- }
- }
- }
- }
- }
- logger.info("Cannot find the given MonitorID in the queue with userName " +
- monitorID.getUserName() + " and jobID " + monitorID.getJobID());
- logger.info("This might not be an error because someone else removed this job from the queue");
- }
-
-
- public static void invokeOutFlowHandlers(JobExecutionContext jobExecutionContext) throws GFacException {
- List<GFacHandlerConfig> handlers = jobExecutionContext.getGFacConfiguration().getOutHandlers();
-
- for (GFacHandlerConfig handlerClassName : handlers) {
- Class<? extends GFacHandler> handlerClass;
- GFacHandler handler;
- try {
- handlerClass = Class.forName(handlerClassName.getClassName().trim()).asSubclass(GFacHandler.class);
- handler = handlerClass.newInstance();
- handler.initProperties(handlerClassName.getProperties());
- } catch (ClassNotFoundException e) {
- logger.error(e.getMessage());
- throw new GFacException("Cannot load handler class " + handlerClassName, e);
- } catch (InstantiationException e) {
- logger.error(e.getMessage());
- throw new GFacException("Cannot instantiate handler class " + handlerClassName, e);
- } catch (IllegalAccessException e) {
- logger.error(e.getMessage());
- throw new GFacException("Cannot instantiate handler class " + handlerClassName, e);
- }
- try {
- handler.invoke(jobExecutionContext);
- } catch (Exception e) {
- // TODO: Better error reporting.
- throw new GFacException("Error Executing a OutFlow Handler", e);
- }
- }
- }
-
- /**
- * Update job count for a given set of paths.
- * @param zk - zookeeper instance
- * @param changeCountMap - map of change job count with relevant path
- * @param isAdd - Should add or reduce existing job count by the given job count.
- */
- public static void updateZkWithJobCount(ZooKeeper zk, final Map<String, Integer> changeCountMap, boolean isAdd) {
- StringBuilder changeZNodePaths = new StringBuilder();
- try {
- if (zk == null || !zk.getState().isConnected()) {
- try {
- final CountDownLatch countDownLatch = new CountDownLatch(1);
- zk = new ZooKeeper(AiravataZKUtils.getZKhostPort(), AiravataZKUtils.getZKTimeout(), new Watcher() {
- @Override
- public void process(WatchedEvent event) {
- countDownLatch.countDown();
- }
- });
- countDownLatch.await();
- } catch (ApplicationSettingsException e) {
- logger.error("Error while reading zookeeper hostport string");
- } catch (IOException e) {
- logger.error("Error while reconnect attempt to zookeeper where zookeeper connection loss state");
- }
- }
-
- for (String path : changeCountMap.keySet()) {
- if (isAdd) {
- CommonUtils.checkAndCreateZNode(zk, path);
- }
- byte[] byteData = zk.getData(path, null, null);
- String nodeData;
- if (byteData == null) {
- if (isAdd) {
- zk.setData(path, String.valueOf(changeCountMap.get(path)).getBytes(), -1);
- } else {
- // This is not possible, but we handle in case there any data zookeeper communication failure
- logger.warn("Couldn't reduce job count in " + path + " as it returns null data. Hence reset the job count to 0");
- zk.setData(path, "0".getBytes(), -1);
- }
- } else {
- nodeData = new String(byteData);
- if (isAdd) {
- zk.setData(path, String.valueOf(changeCountMap.get(path) + Integer.parseInt(nodeData)).getBytes(), -1);
- } else {
- int previousCount = Integer.parseInt(nodeData);
- int removeCount = changeCountMap.get(path);
- if (previousCount >= removeCount) {
- zk.setData(path, String.valueOf(previousCount - removeCount).getBytes(), -1);
- } else {
- // This is not possible, do we need to reset the job count to 0 ?
- logger.error("Requested remove job count is " + removeCount +
- " which is higher than the existing job count " + previousCount
- + " in " + path + " path.");
- }
- }
- }
- changeZNodePaths.append(path).append(":");
- }
-
- // update stat node to trigger orchestrator watchers
- if (changeCountMap.size() > 0) {
- changeZNodePaths.deleteCharAt(changeZNodePaths.length() - 1);
- zk.setData("/" + Constants.STAT, changeZNodePaths.toString().getBytes(), -1);
- }
- } catch (KeeperException e) {
- logger.error("Error while writing job count to zookeeper", e);
- } catch (InterruptedException e) {
- logger.error("Error while writing job count to zookeeper", e);
- }
-
- }
-
- /**
- * Increase job count by one and update the zookeeper
- * @param monitorID - Job monitorId
- */
- public static void increaseZkJobCount(MonitorID monitorID) {
- Map<String, Integer> addMap = new HashMap<String, Integer>();
- addMap.put(CommonUtils.getJobCountUpdatePath(monitorID), 1);
- updateZkWithJobCount(monitorID.getJobExecutionContext().getZk(), addMap, true);
- }
-
- /**
- * Construct and return the path for a given MonitorID , eg: /stat/{username}/{resourceName}/job
- * @param monitorID - Job monitorId
- * @return
- */
- public static String getJobCountUpdatePath(MonitorID monitorID){
- return new StringBuilder("/").append(Constants.STAT).append("/").append(monitorID.getUserName())
- .append("/").append(monitorID.getComputeResourceDescription().getHostName()).append("/").append(Constants.JOB).toString();
- }
-
- /**
- * Check whether znode is exist in given path if not create a new znode
- * @param zk - zookeeper instance
- * @param path - path to check znode
- * @throws KeeperException
- * @throws InterruptedException
- */
- private static void checkAndCreateZNode(ZooKeeper zk , String path) throws KeeperException, InterruptedException {
- if (zk.exists(path, null) == null) { // if znode doesn't exist
- if (path.lastIndexOf("/") > 1) { // recursively traverse to parent znode and check parent exist
- checkAndCreateZNode(zk, (path.substring(0, path.lastIndexOf("/"))));
- }
- zk.create(path, null, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);// create a znode
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/X509Helper.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/X509Helper.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/X509Helper.java
deleted file mode 100644
index 08c3f67..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/util/X509Helper.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.monitor.util;
-
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.ServerSettings;
-import org.bouncycastle.jce.provider.BouncyCastleProvider;
-
-
-import java.io.*;
-import java.security.*;
-import java.security.cert.CertificateException;
-import java.security.cert.CertificateFactory;
-import java.security.cert.CertificateParsingException;
-import java.security.cert.X509Certificate;
-import java.security.spec.InvalidKeySpecException;
-
-public class X509Helper {
-
- static {
- // parsing of RSA key fails without this
- java.security.Security.addProvider(new BouncyCastleProvider());
- }
-
-
-
- public static KeyStore keyStoreFromPEM(String proxyFile,
- String keyPassPhrase) throws IOException,
- CertificateException,
- NoSuchAlgorithmException,
- InvalidKeySpecException,
- KeyStoreException {
- return keyStoreFromPEM(proxyFile,proxyFile,keyPassPhrase);
- }
-
- public static KeyStore keyStoreFromPEM(String certFile,
- String keyFile,
- String keyPassPhrase) throws IOException,
- CertificateException,
- NoSuchAlgorithmException,
- InvalidKeySpecException,
- KeyStoreException {
- CertificateFactory cf = CertificateFactory.getInstance("X.509");
- X509Certificate cert = (X509Certificate)cf.generateCertificate(new FileInputStream(certFile));
- //System.out.println(cert.toString());
-
- // this works for proxy files, too, since it skips over the certificate
- BufferedReader reader = new BufferedReader(new FileReader(keyFile));
- String line = null;
- StringBuilder builder = new StringBuilder();
- boolean inKey = false;
- while((line=reader.readLine()) != null) {
- if (line.contains("-----BEGIN RSA PRIVATE KEY-----")) {
- inKey = true;
- }
- if (inKey) {
- builder.append(line);
- builder.append(System.getProperty("line.separator"));
- }
- if (line.contains("-----END RSA PRIVATE KEY-----")) {
- inKey = false;
- }
- }
- String privKeyPEM = builder.toString();
- //System.out.println(privKeyPEM);
-
- // using BouncyCastle
-// PEMReader pemParser = new PEMReader(new StringReader(privKeyPEM));
-// Object object = pemParser.readObject();
-//
-// PrivateKey privKey = null;
-// if(object instanceof KeyPair){
-// privKey = ((KeyPair)object).getPrivate();
-// }
- // PEMParser from BouncyCastle is good for reading PEM files, but I didn't want to add that dependency
- /*
- // Base64 decode the data
- byte[] encoded = javax.xml.bind.DatatypeConverter.parseBase64Binary(privKeyPEM);
-
- // PKCS8 decode the encoded RSA private key
- java.security.spec.PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(encoded);
- KeyFactory kf = KeyFactory.getInstance("RSA");
- PrivateKey privKey = kf.generatePrivate(keySpec);
- //RSAPrivateKey privKey = (RSAPrivateKey)kf.generatePrivate(keySpec);
- */
- //System.out.println(privKey.toString());
-
-// KeyStore keyStore = KeyStore.getInstance("PKCS12");
-// keyStore.load(null,null);
-//
-// KeyStore.PrivateKeyEntry entry =
-// new KeyStore.PrivateKeyEntry(privKey,
-// new java.security.cert.Certificate[] {(java.security.cert.Certificate)cert});
-// KeyStore.PasswordProtection prot = new KeyStore.PasswordProtection(keyPassPhrase.toCharArray());
-// keyStore.setEntry(cert.getSubjectX500Principal().getName(), entry, prot);
-
-// return keyStore;
- //TODO: Problem with BouncyCastle version used in gsissh
- throw new CertificateException("Method not implemented");
-
- }
-
-
- public static KeyStore trustKeyStoreFromCertDir() throws IOException,
- KeyStoreException,
- CertificateException,
- NoSuchAlgorithmException, ApplicationSettingsException {
- return trustKeyStoreFromCertDir(ServerSettings.getSetting("trusted.cert.location"));
- }
-
- public static KeyStore trustKeyStoreFromCertDir(String certDir) throws IOException,
- KeyStoreException,
- CertificateException,
- NoSuchAlgorithmException {
- KeyStore ks = KeyStore.getInstance("JKS");
- ks.load(null,null);
-
- File dir = new File(certDir);
- for(File file : dir.listFiles()) {
- if (!file.isFile()) {
- continue;
- }
- if (!file.getName().endsWith(".0")) {
- continue;
- }
-
- try {
- //System.out.println("reading file "+file.getName());
- CertificateFactory cf = CertificateFactory.getInstance("X.509");
- X509Certificate cert = (X509Certificate) cf.generateCertificate(new FileInputStream(file));
- //System.out.println(cert.toString());
-
- KeyStore.TrustedCertificateEntry entry = new KeyStore.TrustedCertificateEntry(cert);
-
- ks.setEntry(cert.getSubjectX500Principal().getName(), entry, null);
- } catch (KeyStoreException e) {
- } catch (CertificateParsingException e) {
- continue;
- }
-
- }
-
- return ks;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/errors.properties
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/errors.properties b/modules/gfac/gfac-monitor/src/main/resources/errors.properties
deleted file mode 100644
index 88c41b8..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/errors.properties
+++ /dev/null
@@ -1,197 +0,0 @@
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# Directly copied from jglobus. Not a good way to manager error properties.
-1 = Parameter not supported
-2 = The RSL length is greater than the maximum allowed
-3 = No resources available
-4 = Bad directory specified
-5 = The executable does not exist
-6 = Insufficient funds
-7 = Authentication with the remote server failed
-8 = Job cancelled by user
-9 = Job cancelled by system
-
-10 = Data transfer to the server failed
-11 = The stdin file does not exist
-12 = The connection to the server failed (check host and port)
-13 = The provided RSL 'maxtime' value is invalid (not an integer or must be greater than 0)
-14 = The provided RSL 'count' value is invalid (not an integer or must be greater than 0)
-15 = The job manager received an invalid RSL
-16 = Could not connect to job manager
-17 = The job failed when the job manager attempted to run it
-18 = Paradyn error
-19 = The provided RSL 'jobtype' value is invalid
-
-20 = The provided RSL 'myjob' value is invalid
-21 = The job manager failed to locate an internal script argument file
-22 = The job manager failed to create an internal script argument file
-23 = The job manager detected an invalid job state
-24 = The job manager detected an invalid script response
-25 = The job manager detected an invalid job state
-26 = The provided RSL 'jobtype' value is not supported by this job manager
-27 = Unimplemented
-28 = The job manager failed to create an internal script submission file
-29 = The job manager cannot find the user proxy
-
-30 = The job manager failed to open the user proxy
-31 = The job manager failed to cancel the job as requested
-32 = System memory allocation failed
-33 = The interprocess job communication initialization failed
-34 = The interprocess job communication setup failed
-35 = The provided RSL 'host count' value is invalid
-36 = One of the provided RSL parameters is unsupported
-37 = The provided RSL 'queue' parameter is invalid
-38 = The provided RSL 'project' parameter is invalid
-39 = The provided RSL string includes variables that could not be identified
-
-40 = The provided RSL 'environment' parameter is invalid
-41 = The provided RSL 'dryrun' parameter is invalid
-42 = The provided RSL is invalid (an empty string)
-43 = The job manager failed to stage the executable
-44 = The job manager failed to stage the stdin file
-45 = The requested job manager type is invalid
-46 = The provided RSL 'arguments' parameter is invalid
-47 = The gatekeeper failed to run the job manager
-48 = The provided RSL could not be properly parsed
-49 = There is a version mismatch between GRAM components
-
-50 = The provided RSL 'arguments' parameter is invalid
-51 = The provided RSL 'count' parameter is invalid
-52 = The provided RSL 'directory' parameter is invalid
-53 = The provided RSL 'dryrun' parameter is invalid
-54 = The provided RSL 'environment' parameter is invalid
-55 = The provided RSL 'executable' parameter is invalid
-56 = The provided RSL 'host_count' parameter is invalid
-57 = The provided RSL 'jobtype' parameter is invalid
-58 = The provided RSL 'maxtime' parameter is invalid
-59 = The provided RSL 'myjob' parameter is invalid
-
-60 = The provided RSL 'paradyn' parameter is invalid
-61 = The provided RSL 'project' parameter is invalid
-62 = The provided RSL 'queue' parameter is invalid
-63 = The provided RSL 'stderr' parameter is invalid
-64 = The provided RSL 'stdin' parameter is invalid
-65 = The provided RSL 'stdout' parameter is invalid
-66 = The job manager failed to locate an internal script
-67 = The job manager failed on the system call pipe()
-68 = The job manager failed on the system call fcntl()
-69 = The job manager failed to create the temporary stdout filename
-
-70 = The job manager failed to create the temporary stderr filename
-71 = The job manager failed on the system call fork()
-72 = The executable file permissions do not allow execution
-73 = The job manager failed to open stdout
-74 = The job manager failed to open stderr
-75 = The cache file could not be opened in order to relocate the user proxy
-76 = Cannot access cache files in ~/.globus/.gass_cache, check permissions, quota, and disk space
-77 = The job manager failed to insert the contact in the client contact list
-78 = The contact was not found in the job manager's client contact list
-79 = Connecting to the job manager failed. Possible reasons: job terminated, invalid job contact, network problems, ...
-
-80 = The syntax of the job contact is invalid
-81 = The executable parameter in the RSL is undefined
-82 = The job manager service is misconfigured. condor arch undefined
-83 = The job manager service is misconfigured. condor os undefined
-84 = The provided RSL 'min_memory' parameter is invalid
-85 = The provided RSL 'max_memory' parameter is invalid
-86 = The RSL 'min_memory' value is not zero or greater
-87 = The RSL 'max_memory' value is not zero or greater
-88 = The creation of a HTTP message failed
-89 = Parsing incoming HTTP message failed
-
-90 = The packing of information into a HTTP message failed
-91 = An incoming HTTP message did not contain the expected information
-92 = The job manager does not support the service that the client requested
-93 = The gatekeeper failed to find the requested service
-94 = The jobmanager does not accept any new requests (shutting down)
-95 = The client failed to close the listener associated with the callback URL
-96 = The gatekeeper contact cannot be parsed
-97 = The job manager could not find the 'poe' command
-98 = The job manager could not find the 'mpirun' command
-99 = The provided RSL 'start_time' parameter is invalid"
-100 = The provided RSL 'reservation_handle' parameter is invalid
-
-101 = The provided RSL 'max_wall_time' parameter is invalid
-102 = The RSL 'max_wall_time' value is not zero or greater
-103 = The provided RSL 'max_cpu_time' parameter is invalid
-104 = The RSL 'max_cpu_time' value is not zero or greater
-105 = The job manager is misconfigured, a scheduler script is missing
-106 = The job manager is misconfigured, a scheduler script has invalid permissions
-107 = The job manager failed to signal the job
-108 = The job manager did not recognize/support the signal type
-109 = The job manager failed to get the job id from the local scheduler
-
-110 = The job manager is waiting for a commit signal
-111 = The job manager timed out while waiting for a commit signal
-112 = The provided RSL 'save_state' parameter is invalid
-113 = The provided RSL 'restart' parameter is invalid
-114 = The provided RSL 'two_phase' parameter is invalid
-115 = The RSL 'two_phase' value is not zero or greater
-116 = The provided RSL 'stdout_position' parameter is invalid
-117 = The RSL 'stdout_position' value is not zero or greater
-118 = The provided RSL 'stderr_position' parameter is invalid
-119 = The RSL 'stderr_position' value is not zero or greater
-
-120 = The job manager restart attempt failed
-121 = The job state file doesn't exist
-122 = Could not read the job state file
-123 = Could not write the job state file
-124 = The old job manager is still alive
-125 = The job manager state file TTL expired
-126 = It is unknown if the job was submitted
-127 = The provided RSL 'remote_io_url' parameter is invalid
-128 = Could not write the remote io url file
-129 = The standard output/error size is different
-
-130 = The job manager was sent a stop signal (job is still running)
-131 = The user proxy expired (job is still running)
-132 = The job was not submitted by original jobmanager
-133 = The job manager is not waiting for that commit signal
-134 = The provided RSL scheduler specific parameter is invalid
-135 = The job manager could not stage in a file
-136 = The scratch directory could not be created
-137 = The provided 'gass_cache' parameter is invalid
-138 = The RSL contains attributes which are not valid for job submission
-139 = The RSL contains attributes which are not valid for stdio update
-
-140 = The RSL contains attributes which are not valid for job restart
-141 = The provided RSL 'file_stage_in' parameter is invalid
-142 = The provided RSL 'file_stage_in_shared' parameter is invalid
-143 = The provided RSL 'file_stage_out' parameter is invalid
-144 = The provided RSL 'gass_cache' parameter is invalid
-145 = The provided RSL 'file_cleanup' parameter is invalid
-146 = The provided RSL 'scratch_dir' parameter is invalid
-147 = The provided scheduler-specific RSL parameter is invalid
-148 = A required RSL attribute was not defined in the RSL spec
-149 = The gass_cache attribute points to an invalid cache directory
-
-150 = The provided RSL 'save_state' parameter has an invalid value
-151 = The job manager could not open the RSL attribute validation file
-152 = The job manager could not read the RSL attribute validation file
-153 = The provided RSL 'proxy_timeout' is invalid
-154 = The RSL 'proxy_timeout' value is not greater than zero
-155 = The job manager could not stage out a file
-156 = The job contact string does not match any which the job manager is handling
-157 = Proxy delegation failed
-158 = The job manager could not lock the state lock file
-
-1000 = Failed to start up callback handler
-1003 = Job contact not set
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/AccessPolicy.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/AccessPolicy.json b/modules/gfac/gfac-monitor/src/main/resources/schema/AccessPolicy.json
deleted file mode 100644
index 8f6cfe1..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/AccessPolicy.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/AccessPolicy.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Policy.json"}],
- "properties": {
- "EndpointID": {
- "type": "string",
- "description": "The ID of the Endpoint this AccessPolicy is for"
- }
- },
- "required": ["EndpointID"]
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/Activity.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/Activity.json b/modules/gfac/gfac-monitor/src/main/resources/schema/Activity.json
deleted file mode 100644
index 8bd2495..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/Activity.json
+++ /dev/null
@@ -1,31 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Activity.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
- "properties": {
- "UserDomainID": {
- "type": "string",
- "description": "An ID"
- },
- "EndpointID": {
- "type": "string",
- "description": "The ID of the Endpoint managing Activity"
- },
- "ShareID": {
- "type": "string",
- "description": "The ID of the Share servicing this Activity"
- },
- "ResourceID": {
- "type": "string",
- "description": "The ID of the Resource executing this Activity"
- },
- "ActivityID": {
- "type": "array",
- "description": "The IDs of other Activities related to this one",
- "items": {
- "type": "string"
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/AdminDomain.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/AdminDomain.json b/modules/gfac/gfac-monitor/src/main/resources/schema/AdminDomain.json
deleted file mode 100644
index 8ed4606..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/AdminDomain.json
+++ /dev/null
@@ -1,51 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/AdminDomain.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Domain.json"}],
- "properties": {
- "Distributed": {
- "type": "boolean",
- "description": "true if the services managed by the AdminDomain are geographically distributed"
- },
- "Owner": {
- "type": "array",
- "description": "Identification of persons or legal entities that own the resources in this AdminDomain",
- "items": {
- "type": "string"
- }
- },
- "ServiceID": {
- "type": "array",
- "description": "IDs of Services in this AdminDomain",
- "items": {
- "type": "string"
- }
- },
- "ChildDomainID": {
- "type": "array",
- "description": "IDs of AdminDomains aggregated by this AdminDomain",
- "items": {
- "type": "string"
- }
- },
- "ParentDomainID": {
- "type": "string",
- "description": "The ID of the AdminDomain that this AdminDomain participates in"
- },
- "ComputingServiceID": {
- "type": "array",
- "description": "IDs of ComputingServices in this AdminDomain",
- "items": {
- "type": "string"
- }
- },
- "StorageServiceID": {
- "type": "array",
- "description": "IDs of StorageServices in this AdminDomain",
- "items": {
- "type": "string"
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/ApplicationEnvironment.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/ApplicationEnvironment.json b/modules/gfac/gfac-monitor/src/main/resources/schema/ApplicationEnvironment.json
deleted file mode 100644
index 89c78e0..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/ApplicationEnvironment.json
+++ /dev/null
@@ -1,86 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ApplicationEnvironment.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
- "properties": {
- "AppName": {
- "type": "string",
- "description": "The name of the application"
- },
- "AppVersion": {
- "type": "string",
- "description": "The version of the application"
- },
- "State": {
- "type": "string",
- "description": "The current installation state of the application - AppEnvState_t"
- },
- "RemovalDate": {
- "type": "string",
- "description": "The date/time after which the application may be removed - DateTime_t"
- },
- "License": {
- "type": "string",
- "description": "The license under which the application is usable - License_t"
- },
- "Description": {
- "type": "string",
- "description": "A human-readable description of the application"
- },
- "BestBenchmark": {
- "type": "array",
- "description": "The type(s) of the benchmarks which best describe the sensitivity of this application to the performance of the ExecutionEnvironment - Benchmark_t",
- "items": {
- "type": "string"
- }
- },
- "ParallelSupport": {
- "type": "string",
- "description": "The type of supported parallel execution - ParallelSupport_t"
- },
- "MaxSlots": {
- "type": "integer",
- "description": "The maximum number of concurrent slots that may be used to run the application"
- },
- "MaxJobs": {
- "type": "integer",
- "description": "The maximum number of concurrent jobs that can run the application"
- },
- "MaxUserSeats": {
- "type": "integer",
- "description": "The maximum number of concurrent users that can run the application"
- },
- "FreeSlots": {
- "type": "integer",
- "description": "The maximum number slots currently available to run the application"
- },
- "FreeJobs": {
- "type": "integer",
- "description": "The maximum number of additional jobs that can run the application"
- },
- "FreeUserSeats": {
- "type": "integer",
- "description": "The maximum number of additional users that can run the application"
- },
- "ExecutionEnvironmentID": {
- "type": "array",
- "description": "ID(s) of ExecutionEnvironments where this ApplicationEnvironment can be used",
- "items": {
- "type": "string"
- }
- },
- "ComputingManagerID": {
- "type": "string",
- "description": "ID of the ComputingManager this ApplicationEnvironment is associated with"
- },
- "ApplicationHandleID": {
- "type": "array",
- "description": "ID(s) of the ApplicationHandles that can be used to refer to this environment",
- "items": {
- "type": "string"
- }
- }
- },
- "required": ["AppName","ComputingManagerID"]
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/ApplicationHandle.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/ApplicationHandle.json b/modules/gfac/gfac-monitor/src/main/resources/schema/ApplicationHandle.json
deleted file mode 100644
index e7972e9..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/ApplicationHandle.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ApplicationHandle.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
- "properties": {
- "Type": {
- "type": "string",
- "description": "The type of method used to set up an ApplicationEnvironment - ApplicationHandle_t (open enumeration)"
- },
- "Value": {
- "type": "string",
- "description": "How to set up the ApplicationEnvironment in the context of the Type"
- },
- "ApplicationEnvironmentID": {
- "type": "string",
- "description": "The ID of the ApplicationEnvironment this ApplicationHandle refers to"
- }
- },
- "required": ["Type","Value","ApplicationEnvironmentID"]
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/Benchmark.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/Benchmark.json b/modules/gfac/gfac-monitor/src/main/resources/schema/Benchmark.json
deleted file mode 100644
index 2b64261..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/Benchmark.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Benchmark.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
- "properties": {
- "Type": {
- "type": "string",
- "description": "The type of the benchmark - Benchmark_t (open enumeration)"
- },
- "Value": {
- "type": "number",
- "description": "The value of the benchmark"
- },
- "ComputingManagerID": {
- "type": "string",
- "description": "The ID of the ComputingManager this benchmark is for"
- }
- },
- "required": ["Type","Value"]
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/ComputingActivity.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/ComputingActivity.json b/modules/gfac/gfac-monitor/src/main/resources/schema/ComputingActivity.json
deleted file mode 100644
index 5fcae72..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/ComputingActivity.json
+++ /dev/null
@@ -1,165 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ComputingActivity.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Activity.json"}],
- "properties": {
- "Type": {
- "type": "string",
- "description": "closed enumeration ComputingActivityType_t",
- "enum": ["collectionelement","parallelelement","single","workflownode"]
- },
- "IDFromEndpoint": {
- "type": "string",
- "description": "The ID assigned by the ComputingEndpoint"
- },
- "LocalIDFromManager": {
- "type": "string",
- "description": "The local ID assigned by the ComputingManager"
- },
- "State": {
- "type": "array",
- "description": "open enumeration ComputingActivityState_t",
- "items": {
- "type": "string"
- },
- "minItems": 1
- },
- "RestartState": {
- "type": "array",
- "description": "open enumeration ComputingActivityState_t",
- "items": {
- "type": "string"
- }
- },
- "ExitCode": {
- "type": "integer",
- "description": "The exit code as returned by the main executable code or script of the job"
- },
- "ComputingManagerExitCode": {
- "type": "string",
- "description": "The exit code provided by the ComputingManager"
- },
- "Error": {
- "type": "array",
- "description": "The error messages as provided by the software components involved in the management of the job",
- "items": {
- "type": "string"
- }
- },
- "WaitingPosition": {
- "type": "integer",
- "description": "The position of the job in the queue, if the job is waiting"
- },
- "Owner": {
- "type": "string",
- "description": "The Grid identity of the job's owner"
- },
- "LocalOwner": {
- "type": "string",
- "description": "The local user name of the job's owner"
- },
- "RequestedTotalWallTime": {
- "type": "integer",
- "description": "The total wall clock time requested by the job"
- },
- "RequestedTotalCPUTime": {
- "type": "integer",
- "description": "The total CPU time requested by the job"
- },
- "RequestedSlots": {
- "type": "integer",
- "description": "The number of slots requested for the job"
- },
- "RequestedApplicationEnvironment": {
- "type": "array",
- "description": "The AppName and Version of the requested ApplicationEnvironments",
- "items": {
- "type": "string"
- }
- },
- "StdIn": {
- "type": "string",
- "description": "The name of the file used for standard input"
- },
- "StdOut": {
- "type": "string",
- "description": "The name of the file used for standard output"
- },
- "StdErr": {
- "type": "string",
- "description": "The name of the file used for standard error"
- },
- "LogDir": {
- "type": "string",
- "description": "The name of the directory which contains job logs"
- },
- "ExecutionNode": {
- "type": "array",
- "description": "Hostnames associated with the ExecutionEnvironments running the job",
- "items": {
- "type": "string"
- }
- },
- "Queue": {
- "type": "string",
- "description": "The name of the ComputingManager queue that held the job before execution"
- },
- "UsedTotalWallTime": {
- "type": "integer",
- "description": "The total wall clock time consumed by the job so far (slots*seconds)"
- },
- "UsedTotalCpuTime": {
- "type": "integer",
- "description": "The total CPU time consumed by the job so far (seconds)"
- },
- "UsedMainMemory": {
- "type": "integer",
- "description": "The physical RAM currently used by the job (MB)"
- },
- "SubmissionTime": {
- "type": "string",
- "description": "The time when the job was submitted to the ComputingEndpoint (DateTime_t)"
- },
- "ComputingManagerSubmissionTime": {
- "type": "string",
- "description": "The time when the job was submitted to the ComputingManager (DateTime_t)"
- },
- "StartTime": {
- "type": "string",
- "description": "The time when the ComputingManager started the job (DateTime_t)"
- },
- "EndTime": {
- "type": "string",
- "description": "The time when the job ended in the Grid layer (DateTime_t)"
- },
- "ComputingManagerEndTime": {
- "type": "string",
- "description": "The time when the job ended according to the ComputingManager (DateTime_t)"
- },
- "WorkingAreaEraseTime": {
- "type": "string",
- "description": "The time when working area will be removed from storage (DateTime_t)"
- },
- "ProxyExpirationTime": {
- "type": "string",
- "description": "The expiration time of the Grid proxy associated with the job (DateTime_t)"
- },
- "SubmissionHost": {
- "type": "string",
- "description": "The name of the host from which the job was submitted"
- },
- "SubmissionClientName": {
- "type": "string",
- "description": "The name of the software client used to submit the job"
- },
- "OtherMessages": {
- "type": "array",
- "description": "Optional messages provided by either the Grid layer or the ComputingManager",
- "items": {
- "type": "string"
- }
- }
- },
- "required": ["State","Owner"]
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/ComputingEndpoint.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/ComputingEndpoint.json b/modules/gfac/gfac-monitor/src/main/resources/schema/ComputingEndpoint.json
deleted file mode 100644
index f94f889..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/ComputingEndpoint.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ComputingEndpoint.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Endpoint.json"}],
- "properties": {
- "Staging": {
- "type": "string",
- "description": "Supported file staging functionality - Staging_t",
- "enum": ["none","stagingin","staginginout","stagingout"]
- },
- "JobDescription": {
- "type": "array",
- "description": "Supported job description languages - JobDescription_t (open Enumeration)",
- "items": {
- "type": "string"
- }
- },
- "TotalJobs": {
- "type": "integer",
- "description": "The total number of Grid jobs known to the system"
- },
- "RunningJobs": {
- "type": "integer",
- "description": "The number of Grid jobs which are running in an ExecutionEnvironment"
- },
- "WaitingJobs": {
- "type": "integer",
- "description": "The number of Grid jobs which are waiting to start executing"
- },
- "StagingJobs": {
- "type": "integer",
- "description": "The number of Grid jobs staging files before or after execution"
- },
- "SuspendedJobs": {
- "type": "integer",
- "description": "The number of Grid jobs that started to execute, but are now suspended"
- },
- "PreLRMSWaitingJobs": {
- "type": "integer",
- "description": "The number of Grid jobs managed by the Grid software, but not yet passed to the LRMS"
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/ComputingManager.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/ComputingManager.json b/modules/gfac/gfac-monitor/src/main/resources/schema/ComputingManager.json
deleted file mode 100644
index aecb114..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/ComputingManager.json
+++ /dev/null
@@ -1,117 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ComputingManager.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Share.json"}],
- "properties": {
- "Reservation": {
- "type": "boolean",
- "description": "Whether advance reservation is supported (no value implies undefined in ExtendedBoolean_t)"
- },
- "BulkSubmission": {
- "type": "boolean",
- "description": "Whether multiple jobs can be submitted at once (no value implies undefined in ExtendedBoolean_t)"
- },
- "TotalPhysicalCPUs": {
- "type": "integer",
- "description": "The total number of physical CPUs managed by this ComputingManager"
- },
- "TotalLogicalCPUs": {
- "type": "integer",
- "description": "The total number of logical CPUs managed by this ComputingManager"
- },
- "TotalSlots": {
- "type": "integer",
- "description": "The total number of slots managed by this ComputingManager"
- },
- "SlotsUsedByLocalJobs": {
- "type": "integer",
- "description": "The number of slots currently used by jobs submitted via a non-Grid interface"
- },
- "SlotsUsedByGridJobs": {
- "type": "integer",
- "description": "The number of slots currently used by jobs submitted via a non-Grid interface"
- },
- "Homogeneous": {
- "type": "boolean",
- "description": "Whether this ComputingManager manages only one type of ExecutionEnvironment"
- },
- "NetworkInfo": {
- "type": "array",
- "description": "The types of internal network connections between ExecutionEnvironments (NetworkInfo_t)",
- "items": {
- "type": "string"
- }
- },
- "LocalCPUDistribution": {
- "type": "boolean",
- "description": "Classification of the managed ExecutionEnvironments aggregated by the number of logical CPUs"
- },
- "WorkingAreaShared": {
- "type": "boolean",
- "description": "True if the working area is shared across different ExecutionEnvironments"
- },
- "WorkingAreaGuaranteed": {
- "type": "boolean",
- "description": "True if the job is guaranteed all of WorkingAreaTotal"
- },
- "WorkingAreaTotal": {
- "type": "integer",
- "description": "Total size of the working area available to single slot jobs (GB)"
- },
- "WorkingAreaFree": {
- "type": "integer",
- "description": "The amount of free space in the working area (GB)"
- },
- "WorkingAreaLifeTime": {
- "type": "integer",
- "description": "The minimum guaranteed lifetime of files created in the working area (seconds)"
- },
- "WorkingAreaMultiSlotTotal": {
- "type": "integer",
- "description": "The total size of the working area across all ExecutionEnvironments (GB)"
- },
- "WorkingAreaMultiSlotFree": {
- "type": "integer",
- "description": "The available space in the working area across all ExecutionEnvironments (GB)"
- },
- "WorkingAreaMultiSlotLifeTime": {
- "type": "integer",
- "description": "The minimum guaranteed lifetime of files created in the working area (seconds)"
- },
- "CacheTotal": {
- "type": "integer",
- "description": "If local caching of input files is supported, the total size of the area they may be stored in"
- },
- "CacheFree": {
- "type": "integer",
- "description": "If local caching of input files is supported, the available size of the area they may be stored in"
- },
- "TmpDir": {
- "type": "string",
- "description": "The absolute path of a temporary directory local to an ExecutionEnvironment"
- },
- "ScratchDir": {
- "type": "string",
- "description": "The absolute path of a shared directory available for application data"
- },
- "ApplicationDir": {
- "type": "string",
- "description": "The absolute path of a directory available for installation of persistent application software"
- },
- "ApplicationEnvironmentID": {
- "type": "array",
- "description": "ID(s) of ApplicationEnvironments provided by this ComputingManager",
- "items": {
- "type": "string"
- }
- },
- "BenchmarkID": {
- "type": "array",
- "description": "ID(s) of Benchmarks associated with this ComputingManager",
- "items": {
- "type": "string"
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/ComputingService.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/ComputingService.json b/modules/gfac/gfac-monitor/src/main/resources/schema/ComputingService.json
deleted file mode 100644
index 9cfde1b..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/ComputingService.json
+++ /dev/null
@@ -1,32 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ComputingService.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Service.json"}],
- "properties": {
- "TotalJobs": {
- "type": "integer",
- "description": "The total number of Grid jobs known to the system"
- },
- "RunningJobs": {
- "type": "integer",
- "description": "The number of Grid jobs which are running in an ExecutionEnvironment"
- },
- "WaitingJobs": {
- "type": "integer",
- "description": "The number of Grid jobs which are waiting to start executing"
- },
- "StagingJobs": {
- "type": "integer",
- "description": "The number of Grid jobs staging files before or after execution"
- },
- "SuspendedJobs": {
- "type": "integer",
- "description": "The number of Grid jobs that started to execute, but are now suspended"
- },
- "PreLRMSWaitingJobs": {
- "type": "integer",
- "description": "The number of Grid jobs managed by the Grid software, but not yet passed to the LRMS"
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/ComputingShare.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/ComputingShare.json b/modules/gfac/gfac-monitor/src/main/resources/schema/ComputingShare.json
deleted file mode 100644
index 340c83e..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/ComputingShare.json
+++ /dev/null
@@ -1,182 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/ComputingShare.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Share.json"}],
- "properties": {
- "MappingQueue": {
- "type": "string",
- "description": "The name of the queue in the LRMS where jobs in this share are submitted"
- },
- "MaxWallTime": {
- "type": "integer",
- "description": "The maximum wall clock time that a single-slot job can run (seconds)"
- },
- "MaxMultiSlotWallTime": {
- "type": "integer",
- "description": "The maximum wall clock time that a multi-slot job can run (seconds)"
- },
- "DefaultWallTime": {
- "type": "integer",
- "description": "The default wall clock per slot assumed by the LRMS if a maximum time is not specified (seconds)"
- },
- "MaxCPUTime": {
- "type": "integer",
- "description": "The maximum pre-slot CPU time that a job can request (seconds)"
- },
- "MaxTotalCPUTime": {
- "type": "integer",
- "description": "The maximum amount of CPU time that a job can request across all slots assigned to it (seconds)"
- },
- "MinCPUTime": {
- "type": "integer",
- "description": "The minimum pre-slot CPU time that a job can request (seconds)"
- },
- "DefaultCPUTime": {
- "type": "integer",
- "description": "The default CPU time limit assumed by the LRMS if a maximum time is not specified (seconds)"
- },
- "MaxTotalJobs": {
- "type": "integer",
- "description": "The maximum number of jobs that can be in this Share"
- },
- "MaxRunningJobs": {
- "type": "integer",
- "description": "The maximum number of jobs that can be running in this Share"
- },
- "MaxWaitingJobs": {
- "type": "integer",
- "description": "The maximum number of jobs that can be waiting in this Share"
- },
- "MaxPreLRMSWaitingJobs": {
- "type": "integer",
- "description": "The maximum number of jobs that can be waiting in the Grid layer for this Share"
- },
- "MaxUserRunningJobs": {
- "type": "integer",
- "description": "The maximum number of jobs that can be running in this Share per user"
- },
- "MaxSlotsPerJob": {
- "type": "integer",
- "description": "The maximum number of slots that can be allocated to a single job in this Share"
- },
- "MaxStageInStreams": {
- "type": "integer",
- "description": "The maximum number of streams available to stage files in"
- },
- "MaxStageOutStreams": {
- "type": "integer",
- "description": "The maximum number of streams available to stage files out"
- },
- "ScheduingPolicy": {
- "type": "string",
- "description": "The scheduling policy used by the share - SchedulingPolicy_t (open enumeration)"
- },
- "MaxMainMemory": {
- "type": "integer",
- "description": "The maximum amount of physical RAM that a job can use (MB)"
- },
- "GuaranteedMainMemory": {
- "type": "integer",
- "description": "The amount of physical RAM that a job will have available (MB)"
- },
- "MaxVirtualMemory": {
- "type": "integer",
- "description": "The maximum amount memory (RAM+swap) that a job can use (MB)"
- },
- "GuaranteedVirtualMemory": {
- "type": "integer",
- "description": "The amount of memory (RAM+swap) that a job will have available (MB)"
- },
- "MaxDiskSpace": {
- "type": "integer",
- "description": "The maximum disk space that a job can use in the working area (GB)"
- },
- "DefaultStorageServiceID": {
- "type": "string",
- "description": "The ID of the default StorageService used to store files"
- },
- "Preemption": {
- "type": "boolean",
- "description": "Whether jobs can be preempted and resumed (no value implies undefined in ExtendedBoolean_t)"
- },
- "ServingState": {
- "type": "string",
- "description": "How the Share is currently serving jobs",
- "enum": ["closed","draining","production","queueing"]
- },
- "TotalJobs": {
- "type": "integer",
- "description": "The total number of jobs in any state"
- },
- "RunningJobs": {
- "type": "integer",
- "description": "The number of running jobs submitted through Grid or non-Grid interfaces"
- },
- "LocalRunningJobs": {
- "type": "integer",
- "description": "The number of running jobs submitted using non-Grid interfaces"
- },
- "WaitingJobs": {
- "type": "integer",
- "description": "The number of waiting jobs submitted through Grid or non-Grid interfaces"
- },
- "LocalWaitingJobs": {
- "type": "integer",
- "description": "The number of waiting jobs submitted using non-Grid interfaces"
- },
- "SuspendedJobs": {
- "type": "integer",
- "description": "The number of suspended jobs submitted through Grid or non-Grid interfaces"
- },
- "LocalSuspendedJobs": {
- "type": "integer",
- "description": "The number of suspended jobs submitted using non-Grid interfaces"
- },
- "StagingJobs": {
- "type": "integer",
- "description": "The number of jobs staging files before or after execution"
- },
- "PreLRMSWaitingJobs": {
- "type": "integer",
- "description": "The number of Grid jobs which have not yet been passed to the LRMS"
- },
- "EstimatedAverageWaitingTime": {
- "type": "integer",
- "description": "An estimate of the average time a job will wait before it starts to execute (seconds)"
- },
- "EstimatedWorstWaitingTime": {
- "type": "integer",
- "description": "An estimate of the worst-case time a job will wait before it starts to execute (seconds)"
- },
- "FreeSlots": {
- "type": "integer",
- "description": "The number of slots which are currently available for use"
- },
- "FreeSlotsWithDuration": {
- "type": "string",
- "description": "The number of slots which are currently available for use and how long they are available"
- },
- "UsedSlots": {
- "type": "integer",
- "description": "The number of slots currently in use"
- },
- "RequestedSlots": {
- "type": "integer",
- "description": "The number of slots needd to execute all waiting and staging jobs"
- },
- "ReservationPolicy": {
- "type": "string",
- "description": "The policy used for advance reservation - ReservationPolicy_t",
- "enum": ["mandatory","none","optional"]
- },
- "Tag": {
- "type": "array",
- "description": "UserDomain-defined tags for this Share",
- "items": {
- "type": "string"
- }
- }
- },
- "required": ["ServingState"]
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/Contact.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/Contact.json b/modules/gfac/gfac-monitor/src/main/resources/schema/Contact.json
deleted file mode 100644
index 436b262..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/Contact.json
+++ /dev/null
@@ -1,32 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Contact.json",
- "description": "A GLUE 2 Contact",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
- "properties": {
- "Detail": {
- "type": "string",
- "description": "A URI embedding the contact information"
- },
- "Type": {
- "type": "string",
- "description": "closed enumeration ContactType_t",
- "enum": ["general","security","sysadmin","usersupport"]
- },
- "ServiceID": {
- "type": "array",
- "description": "The IDs of Services associated with this Contact",
- "items": {
- "type": "string"
- }
- },
- "DomainID": {
- "type": "array",
- "description": "The IDs of Domains associated with this Contact",
- "items": {
- "type": "string"
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/DataStore.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/DataStore.json b/modules/gfac/gfac-monitor/src/main/resources/schema/DataStore.json
deleted file mode 100644
index 8f15447..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/DataStore.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/DataStore.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Resource.json"}],
- "properties": {
- "Type": {
- "type": "string",
- "description": "The type of storage medium - DataStoreType_t (disk,optical,tape,...)"
- },
- "Latency": {
- "type": "string",
- "description": "The latency category under normal operating conditions - AccessLatency_t",
- "enum": ["nearline","offline","online"]
- },
- "TotalSize": {
- "type": "integer",
- "description": "The total amount of storage (GB)"
- },
- "FreeSize": {
- "type": "integer",
- "description": "The amount of available storage (GB)"
- },
- "UsedSize": {
- "type": "integer",
- "description": "The amount of used storage (GB)"
- }
- },
- "required": ["Type","Latency"]
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/resources/schema/Domain.json
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/resources/schema/Domain.json b/modules/gfac/gfac-monitor/src/main/resources/schema/Domain.json
deleted file mode 100644
index 5bd996b..0000000
--- a/modules/gfac/gfac-monitor/src/main/resources/schema/Domain.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
- "$schema": "http://json-schema.org/draft-04/schema#",
- "id": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Domain.json",
- "type": "object",
- "allOf": [{"$ref": "http://schemas.ogf.org/glue/2013/05/spec_2.0_r1/Entity.json"}],
- "properties": {
- "Description": {
- "type": "string",
- "description": "A description of the Domain"
- },
- "WWW": {
- "type": "array",
- "description": "URLs of web pages with more information about the Domain",
- "items": {
- "type": "string"
- }
- },
- "ContactID": {
- "type": "array",
- "description": "IDs of Contacts for this Domain",
- "items": {
- "type": "string"
- }
- },
- "LocationID": {
- "type": "string",
- "description": "The ID of the primary Location for this Domain"
- }
- }
-}
[3/7] airavata git commit: Add new gfac email monitor module to gfac
monitor module and Intorduced gfac-hpc-monitor module to keep all hoc-monitor
code
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/HostMonitorData.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/HostMonitorData.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/HostMonitorData.java
deleted file mode 100644
index f29e3e6..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/HostMonitorData.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.monitor;
-
-import org.apache.airavata.gfac.core.context.JobExecutionContext;
-import org.apache.airavata.gfac.core.monitor.MonitorID;
-import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
-import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
-import org.apache.airavata.model.appcatalog.computeresource.DataMovementProtocol;
-import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class HostMonitorData {
-// private HostDescription host;
- private ComputeResourceDescription computeResourceDescription;
- private JobSubmissionProtocol jobSubmissionProtocol;
- private DataMovementProtocol dataMovementProtocol;
-
- private List<MonitorID> monitorIDs;
-
- public HostMonitorData(JobExecutionContext jobExecutionContext) {
- this.computeResourceDescription = jobExecutionContext.getApplicationContext().getComputeResourceDescription();
- this.jobSubmissionProtocol = jobExecutionContext.getPreferredJobSubmissionProtocol();
- this.dataMovementProtocol = jobExecutionContext.getPreferredDataMovementProtocol();
- this.monitorIDs = new ArrayList<MonitorID>();
- }
-
- public HostMonitorData(JobExecutionContext jobExecutionContext, List<MonitorID> monitorIDs) {
- this.computeResourceDescription = jobExecutionContext.getApplicationContext().getComputeResourceDescription();
- this.jobSubmissionProtocol = jobExecutionContext.getPreferredJobSubmissionProtocol();
- this.dataMovementProtocol = jobExecutionContext.getPreferredDataMovementProtocol();
- this.monitorIDs = monitorIDs;
- }
-
- public ComputeResourceDescription getComputeResourceDescription() {
- return computeResourceDescription;
- }
-
- public void setComputeResourceDescription(ComputeResourceDescription computeResourceDescription) {
- this.computeResourceDescription = computeResourceDescription;
- }
-
- public List<MonitorID> getMonitorIDs() {
- return monitorIDs;
- }
-
- public void setMonitorIDs(List<MonitorID> monitorIDs) {
- this.monitorIDs = monitorIDs;
- }
-
- /**
- * this method get called by CommonUtils and it will check the right place before adding
- * so there will not be a mismatch between this.host and monitorID.host
- * @param monitorID
- * @throws org.apache.airavata.gfac.monitor.exception.AiravataMonitorException
- */
- public void addMonitorIDForHost(MonitorID monitorID)throws AiravataMonitorException {
- monitorIDs.add(monitorID);
- }
-
- public JobSubmissionProtocol getJobSubmissionProtocol() {
- return jobSubmissionProtocol;
- }
-
- public DataMovementProtocol getDataMovementProtocol() {
- return dataMovementProtocol;
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/UserMonitorData.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/UserMonitorData.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/UserMonitorData.java
deleted file mode 100644
index 022d17c..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/UserMonitorData.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.monitor;
-
-import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * This is the datastructure to keep the user centric job data, rather keeping
- * the individual jobs we keep the jobs based on the each user
- */
-public class UserMonitorData {
- private final static Logger logger = LoggerFactory.getLogger(UserMonitorData.class);
-
- private String userName;
-
- private List<HostMonitorData> hostMonitorData;
-
-
- public UserMonitorData(String userName) {
- this.userName = userName;
- hostMonitorData = new ArrayList<HostMonitorData>();
- }
-
- public UserMonitorData(String userName, List<HostMonitorData> hostMonitorDataList) {
- this.hostMonitorData = hostMonitorDataList;
- this.userName = userName;
- }
-
- public List<HostMonitorData> getHostMonitorData() {
- return hostMonitorData;
- }
-
- public void setHostMonitorData(List<HostMonitorData> hostMonitorData) {
- this.hostMonitorData = hostMonitorData;
- }
-
- public String getUserName() {
- return userName;
- }
-
- public void setUserName(String userName) {
- this.userName = userName;
- }
-
- /*
- This method will add element to the MonitorID list, user should not
- duplicate it, we do not check it because its going to be used by airavata
- so we have to use carefully and this method will add a host if its a new host
- */
- public void addHostMonitorData(HostMonitorData hostMonitorData) throws AiravataMonitorException {
- this.hostMonitorData.add(hostMonitorData);
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/command/ExperimentCancelRequest.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/command/ExperimentCancelRequest.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/command/ExperimentCancelRequest.java
deleted file mode 100644
index f19decf..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/command/ExperimentCancelRequest.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.monitor.command;
-
-public class ExperimentCancelRequest {
- private String experimentId;
-
- public ExperimentCancelRequest(String experimentId) {
- this.experimentId = experimentId;
- }
-
- public String getExperimentId() {
- return experimentId;
- }
-
- public void setExperimentId(String experimentId) {
- this.experimentId = experimentId;
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/command/TaskCancelRequest.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/command/TaskCancelRequest.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/command/TaskCancelRequest.java
deleted file mode 100644
index b45e01c..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/command/TaskCancelRequest.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.monitor.command;
-
-public class TaskCancelRequest {
- private String experimentId;
- private String nodeId;
- private String taskId;
-
- public TaskCancelRequest(String experimentId, String nodeId, String taskId) {
- this.experimentId = experimentId;
- this.setNodeId(nodeId);
- this.taskId = taskId;
- }
- public String getExperimentId() {
- return experimentId;
- }
- public void setExperimentId(String experimentId) {
- this.experimentId = experimentId;
- }
- public String getTaskId() {
- return taskId;
- }
- public void setTaskId(String taskId) {
- this.taskId = taskId;
- }
- public String getNodeId() {
- return nodeId;
- }
- public void setNodeId(String nodeId) {
- this.nodeId = nodeId;
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/AiravataAbstractMonitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/AiravataAbstractMonitor.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/AiravataAbstractMonitor.java
deleted file mode 100644
index c754d3c..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/AiravataAbstractMonitor.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.monitor.core;
-
-import org.apache.airavata.common.utils.MonitorPublisher;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * This is the abstract Monitor which needs to be used by
- * any Monitoring implementation which expect nto consume
- * to store the status to registry. Because they have to
- * use the MonitorPublisher to publish the monitoring statuses
- * to the Event Bus. All the Monitor statuses publish to the eventbus
- * will be saved to the Registry.
- */
-public abstract class AiravataAbstractMonitor implements Monitor {
- private final static Logger logger = LoggerFactory.getLogger(AiravataAbstractMonitor.class);
- protected MonitorPublisher publisher;
-
- public MonitorPublisher getPublisher() {
- return publisher;
- }
-
- public void setPublisher(MonitorPublisher publisher) {
- this.publisher = publisher;
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/MessageParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/MessageParser.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/MessageParser.java
deleted file mode 100644
index a003f55..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/MessageParser.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.monitor.core;
-
-import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
-import org.apache.airavata.model.workspace.experiment.JobState;
-
-/**
- * This is an interface to implement messageparser, it could be
- * pull based or push based still monitor has to parse the content of
- * the message it gets from remote monitoring system and finalize
- * them to internal job state, Ex: JSON parser for AMQP and Qstat reader
- * for pull based monitor.
- */
-public interface MessageParser {
- /**
- * This method is to implement how to parse the incoming message
- * and implement a logic to finalize the status of the job,
- * we have to makesure the correct message is given to the messageparser
- * parse method, it will not do any filtering
- * @param message content of the message
- * @return
- */
- JobState parseMessage(String message)throws AiravataMonitorException;
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/Monitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/Monitor.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/Monitor.java
deleted file mode 100644
index 614d606..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/Monitor.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.monitor.core;
-
-
-/**
- * This is the primary interface for Monitors,
- * This can be used to implement different methods of monitoring
- */
-public interface Monitor {
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/PullMonitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/PullMonitor.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/PullMonitor.java
deleted file mode 100644
index efdf89c..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/PullMonitor.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.monitor.core;
-
-import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
-
-/**
- * PullMonitors can implement this interface
- * Since the pull and push based monitoring required different
- * operations, PullMonitor will be useful.
- * This will allow users to program Pull monitors separately
- */
-public abstract class PullMonitor extends AiravataAbstractMonitor {
-
- private int pollingFrequence;
- /**
- * This method will can invoke when PullMonitor needs to start
- * and it has to invoke in the frequency specified below,
- * @return if the start process is successful return true else false
- */
- public abstract boolean startPulling() throws AiravataMonitorException;
-
- /**
- * This is the method to stop the polling process
- * @return if the stopping process is successful return true else false
- */
- public abstract boolean stopPulling()throws AiravataMonitorException;
-
- /**
- * this method can be used to set the polling frequencey or otherwise
- * can implement a polling mechanism, and implement how to do
- * @param frequence
- */
- public void setPollingFrequence(int frequence){
- this.pollingFrequence = frequence;
- }
-
- /**
- * this method can be used to get the polling frequencey or otherwise
- * can implement a polling mechanism, and implement how to do
- * @return
- */
- public int getPollingFrequence(){
- return this.pollingFrequence;
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/PushMonitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/PushMonitor.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/PushMonitor.java
deleted file mode 100644
index 1b6a228..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/core/PushMonitor.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.monitor.core;
-
-import org.apache.airavata.gfac.core.monitor.MonitorID;
-import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
-
-/**
- * PushMonitors can implement this interface
- * Since the pull and push based monitoring required different
- * operations, PullMonitor will be useful.
- * This interface will allow users to program Push monitors separately
- */
-public abstract class PushMonitor extends AiravataAbstractMonitor {
- /**
- * This method can be invoked to register a listener with the
- * remote monitoring system, ideally inside this method users will be
- * writing some client listener code for the remote monitoring system,
- * this will be a simple wrapper around any client for the remote Monitor.
- * @param monitorID
- * @return
- */
- public abstract boolean registerListener(MonitorID monitorID)throws AiravataMonitorException;
-
- /**
- * This method can be invoked to unregister a listener with the
- * remote monitoring system, ideally inside this method users will be
- * writing some client listener code for the remote monitoring system,
- * this will be a simple wrapper around any client for the remote Monitor.
- * @param monitorID
- * @return
- */
- public abstract boolean unRegisterListener(MonitorID monitorID)throws AiravataMonitorException;
-
- /**
- * This can be used to stop the registration thread
- * @return
- * @throws org.apache.airavata.gfac.monitor.exception.AiravataMonitorException
- */
- public abstract boolean stopRegister()throws AiravataMonitorException;
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/exception/AiravataMonitorException.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/exception/AiravataMonitorException.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/exception/AiravataMonitorException.java
deleted file mode 100644
index 3acef66..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/exception/AiravataMonitorException.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.monitor.exception;
-
-public class AiravataMonitorException extends Exception {
- private static final long serialVersionUID = -2849422320139467602L;
-
- public AiravataMonitorException(Throwable e) {
- super(e);
- }
-
- public AiravataMonitorException(String message) {
- super(message, null);
- }
-
- public AiravataMonitorException(String message, Throwable e) {
- super(message, e);
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java
deleted file mode 100644
index 24b300e..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.monitor.handlers;
-
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.logger.AiravataLogger;
-import org.apache.airavata.common.logger.AiravataLoggerFactory;
-import org.apache.airavata.common.utils.ServerSettings;
-import org.apache.airavata.gfac.core.context.JobExecutionContext;
-import org.apache.airavata.gfac.core.cpi.BetterGfacImpl;
-import org.apache.airavata.gfac.core.handler.GFacHandlerException;
-import org.apache.airavata.gfac.core.handler.ThreadedHandler;
-import org.apache.airavata.gfac.core.monitor.MonitorID;
-import org.apache.airavata.gfac.core.utils.GFacUtils;
-import org.apache.airavata.gfac.monitor.HPCMonitorID;
-import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
-import org.apache.airavata.gfac.monitor.impl.pull.qstat.HPCPullMonitor;
-import org.apache.airavata.gfac.monitor.util.CommonUtils;
-import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
-import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooKeeper;
-import org.apache.zookeeper.data.Stat;
-
-import java.io.File;
-import java.util.Properties;
-
-/**
- * this handler is responsible for monitoring jobs in a pull mode
- * and currently this support multiple pull monitoring in grid resource and uses
- * commands like qstat,squeue and this supports sun grid enging monitoring too
- * which is a slight variation of qstat monitoring.
- */
-public class GridPullMonitorHandler extends ThreadedHandler implements Watcher{
- private final static AiravataLogger logger = AiravataLoggerFactory.getLogger(GridPullMonitorHandler.class);
-
- private HPCPullMonitor hpcPullMonitor;
-
- private AuthenticationInfo authenticationInfo;
-
- public void initProperties(Properties properties) throws GFacHandlerException {
- String myProxyUser = null;
- try {
- myProxyUser = ServerSettings.getSetting("myproxy.username");
- String myProxyPass = ServerSettings.getSetting("myproxy.password");
- String certPath = ServerSettings.getSetting("trusted.cert.location");
- String myProxyServer = ServerSettings.getSetting("myproxy.server");
- setAuthenticationInfo(new MyProxyAuthenticationInfo(myProxyUser, myProxyPass, myProxyServer,
- 7512, 17280000, certPath));
- if(BetterGfacImpl.getMonitorPublisher() != null){
- hpcPullMonitor = new HPCPullMonitor(BetterGfacImpl.getMonitorPublisher(),getAuthenticationInfo()); // we use our own credentials for monitoring, not from the store
- }else {
- throw new GFacHandlerException("Error initializing Monitor Handler, because Monitor Publisher is null !!!");
- }
- } catch (ApplicationSettingsException e) {
- logger.error("Error while reading server properties", e);
- throw new GFacHandlerException("Error while reading server properties", e);
- }
- }
-
- public void run() {
- hpcPullMonitor.run();
- }
-
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- super.invoke(jobExecutionContext);
- hpcPullMonitor.setGfac(jobExecutionContext.getGfac());
- MonitorID monitorID = new HPCMonitorID(getAuthenticationInfo(), jobExecutionContext);
- try {
- ZooKeeper zk = jobExecutionContext.getZk();
- try {
- String experimentEntry = GFacUtils.findExperimentEntry(jobExecutionContext.getExperimentID(), zk);
- String path = experimentEntry + File.separator + "operation";
- Stat exists = zk.exists(path, this);
- if (exists != null) {
- zk.getData(path, this, exists); // watching the operations node
- }
- } catch (KeeperException e) {
- logger.error(e.getMessage(), e);
- } catch (InterruptedException e) {
- logger.error(e.getMessage(), e);
- }
- CommonUtils.addMonitortoQueue(hpcPullMonitor.getQueue(), monitorID, jobExecutionContext);
- CommonUtils.increaseZkJobCount(monitorID); // update change job count to zookeeper
- } catch (AiravataMonitorException e) {
- logger.errorId(monitorID.getJobID(), "Error adding job {} monitorID object to the queue with experiment {}",
- monitorID.getJobID(), monitorID.getExperimentID());
- }
- }
- public AuthenticationInfo getAuthenticationInfo() {
- return authenticationInfo;
- }
-
- public HPCPullMonitor getHpcPullMonitor() {
- return hpcPullMonitor;
- }
-
- public void setAuthenticationInfo(AuthenticationInfo authenticationInfo) {
- this.authenticationInfo = authenticationInfo;
- }
-
- public void setHpcPullMonitor(HPCPullMonitor hpcPullMonitor) {
- this.hpcPullMonitor = hpcPullMonitor;
- }
-
-
- public void process(WatchedEvent watchedEvent) {
- logger.info(watchedEvent.getPath());
- if(Event.EventType.NodeDataChanged.equals(watchedEvent.getType())){
- // node data is changed, this means node is cancelled.
- logger.info("Experiment is cancelled with this path:"+watchedEvent.getPath());
-
- String[] split = watchedEvent.getPath().split("/");
- for(String element:split) {
- if (element.contains("+")) {
- logger.info("Adding experimentID+TaskID to be removed from monitoring:"+element);
- hpcPullMonitor.getCancelJobList().add(element);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPushMonitorHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPushMonitorHandler.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPushMonitorHandler.java
deleted file mode 100644
index 0eb4526..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPushMonitorHandler.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.monitor.handlers;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.ServerSettings;
-import org.apache.airavata.gfac.core.context.JobExecutionContext;
-import org.apache.airavata.gfac.core.cpi.BetterGfacImpl;
-import org.apache.airavata.gfac.core.handler.GFacHandlerException;
-import org.apache.airavata.gfac.core.handler.ThreadedHandler;
-import org.apache.airavata.gfac.core.monitor.MonitorID;
-import org.apache.airavata.gfac.monitor.HPCMonitorID;
-import org.apache.airavata.gfac.monitor.impl.push.amqp.AMQPMonitor;
-import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
-import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * this handler is responsible monitoring jobs in push mode
- * and currently this support multiple push monitoring in grid resource
- */
-public class GridPushMonitorHandler extends ThreadedHandler {
- private final static Logger logger= LoggerFactory.getLogger(GridPushMonitorHandler.class);
-
- private AMQPMonitor amqpMonitor;
-
- private AuthenticationInfo authenticationInfo;
-
- @Override
- public void initProperties(Properties properties) throws GFacHandlerException {
- String myProxyUser=null;
- try{
- myProxyUser = ServerSettings.getSetting("myproxy.username");
- String myProxyPass = ServerSettings.getSetting("myproxy.password");
- String certPath = ServerSettings.getSetting("trusted.cert.location");
- String myProxyServer = ServerSettings.getSetting("myproxy.server");
- setAuthenticationInfo(new MyProxyAuthenticationInfo(myProxyUser, myProxyPass, myProxyServer,
- 7512, 17280000, certPath));
-
- String hostList=(String)properties.get("hosts");
- String proxyFilePath = ServerSettings.getSetting("proxy.file.path");
- String connectionName=ServerSettings.getSetting("connection.name");
- LinkedBlockingQueue<MonitorID> pushQueue = new LinkedBlockingQueue<MonitorID>();
- LinkedBlockingQueue<MonitorID> finishQueue = new LinkedBlockingQueue<MonitorID>();
- List<String> hosts= Arrays.asList(hostList.split(","));
- amqpMonitor=new AMQPMonitor(BetterGfacImpl.getMonitorPublisher(),pushQueue,finishQueue,proxyFilePath,connectionName,hosts);
- }catch (ApplicationSettingsException e){
- logger.error(e.getMessage(), e);
- throw new GFacHandlerException(e.getMessage(), e);
- }
- }
-
- @Override
- public void run() {
- amqpMonitor.run();
- }
-
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException{
- super.invoke(jobExecutionContext);
- MonitorID monitorID=new HPCMonitorID(getAuthenticationInfo(),jobExecutionContext);
- amqpMonitor.getRunningQueue().add(monitorID);
- }
-
- public AMQPMonitor getAmqpMonitor() {
- return amqpMonitor;
- }
-
- public void setAmqpMonitor(AMQPMonitor amqpMonitor) {
- this.amqpMonitor = amqpMonitor;
- }
-
- public AuthenticationInfo getAuthenticationInfo() {
- return authenticationInfo;
- }
-
- public void setAuthenticationInfo(AuthenticationInfo authenticationInfo) {
- this.authenticationInfo = authenticationInfo;
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
deleted file mode 100644
index ab934ca..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
+++ /dev/null
@@ -1,469 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.monitor.impl.pull.qstat;
-
-import com.google.common.eventbus.EventBus;
-import org.apache.airavata.common.logger.AiravataLogger;
-import org.apache.airavata.common.logger.AiravataLoggerFactory;
-import org.apache.airavata.common.utils.MonitorPublisher;
-import org.apache.airavata.common.utils.ServerSettings;
-import org.apache.airavata.gfac.monitor.util.CommonUtils;
-import org.apache.airavata.gfac.core.cpi.GFac;
-import org.apache.airavata.gfac.core.monitor.MonitorID;
-import org.apache.airavata.gfac.core.utils.GFacThreadPoolExecutor;
-import org.apache.airavata.gfac.core.utils.OutHandlerWorker;
-import org.apache.airavata.gfac.monitor.HostMonitorData;
-import org.apache.airavata.gfac.monitor.UserMonitorData;
-import org.apache.airavata.gfac.monitor.core.PullMonitor;
-import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
-import org.apache.airavata.gfac.monitor.impl.push.amqp.SimpleJobFinishConsumer;
-import org.apache.airavata.gsi.ssh.api.SSHApiException;
-import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
-import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
-import org.apache.airavata.model.messaging.event.JobIdentifier;
-import org.apache.airavata.model.messaging.event.JobStatusChangeRequestEvent;
-import org.apache.airavata.model.workspace.experiment.JobState;
-
-import java.sql.Timestamp;
-import java.util.*;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingDeque;
-import java.util.concurrent.LinkedBlockingQueue;
-
-/**
- * This monitor is based on qstat command which can be run
- * in grid resources and retrieve the job status.
- */
-public class HPCPullMonitor extends PullMonitor {
-
- private final static AiravataLogger logger = AiravataLoggerFactory.getLogger(HPCPullMonitor.class);
- public static final int FAILED_COUNT = 5;
-
- // I think this should use DelayedBlocking Queue to do the monitoring*/
- private BlockingQueue<UserMonitorData> queue;
-
- private boolean startPulling = false;
-
- private Map<String, ResourceConnection> connections;
-
- private MonitorPublisher publisher;
-
- private LinkedBlockingQueue<String> cancelJobList;
-
- private List<String> completedJobsFromPush;
-
- private GFac gfac;
-
- private AuthenticationInfo authenticationInfo;
-
- private ArrayList<MonitorID> removeList;
-
- public HPCPullMonitor() {
- connections = new HashMap<String, ResourceConnection>();
- queue = new LinkedBlockingDeque<UserMonitorData>();
- publisher = new MonitorPublisher(new EventBus());
- cancelJobList = new LinkedBlockingQueue<String>();
- completedJobsFromPush = new ArrayList<String>();
- (new SimpleJobFinishConsumer(this.completedJobsFromPush)).listen();
- removeList = new ArrayList<MonitorID>();
- }
-
- public HPCPullMonitor(MonitorPublisher monitorPublisher, AuthenticationInfo authInfo) {
- connections = new HashMap<String, ResourceConnection>();
- queue = new LinkedBlockingDeque<UserMonitorData>();
- publisher = monitorPublisher;
- authenticationInfo = authInfo;
- cancelJobList = new LinkedBlockingQueue<String>();
- this.completedJobsFromPush = new ArrayList<String>();
- (new SimpleJobFinishConsumer(this.completedJobsFromPush)).listen();
- removeList = new ArrayList<MonitorID>();
- }
-
- public HPCPullMonitor(BlockingQueue<UserMonitorData> queue, MonitorPublisher publisher) {
- this.queue = queue;
- this.publisher = publisher;
- connections = new HashMap<String, ResourceConnection>();
- cancelJobList = new LinkedBlockingQueue<String>();
- this.completedJobsFromPush = new ArrayList<String>();
- (new SimpleJobFinishConsumer(this.completedJobsFromPush)).listen();
- removeList = new ArrayList<MonitorID>();
- }
-
-
- public void run() {
- /* implement a logic to pick each monitorID object from the queue and do the
- monitoring
- */
- this.startPulling = true;
- while (this.startPulling && !ServerSettings.isStopAllThreads()) {
- try {
- // After finishing one iteration of the full queue this thread sleeps 1 second
- synchronized (this.queue) {
- if (this.queue.size() > 0) {
- startPulling();
- }
- }
- Thread.sleep(10000);
- } catch (Exception e) {
- // we catch all the exceptions here because no matter what happens we do not stop running this
- // thread, but ideally we should report proper error messages, but this is handled in startPulling
- // method, incase something happen in Thread.sleep we handle it with this catch block.
- logger.error(e.getMessage(),e);
- }
- }
- // thread is going to return so we close all the connections
- Iterator<String> iterator = connections.keySet().iterator();
- while (iterator.hasNext()) {
- String next = iterator.next();
- ResourceConnection resourceConnection = connections.get(next);
- try {
- resourceConnection.getCluster().disconnect();
- } catch (SSHApiException e) {
- logger.error("Erro while connecting to the cluster", e);
- }
- }
- }
-
- /**
- * This method will can invoke when PullMonitor needs to start
- * and it has to invoke in the frequency specified below,
- *
- * @return if the start process is successful return true else false
- */
- public boolean startPulling() throws AiravataMonitorException {
- // take the top element in the queue and pull the data and put that element
- // at the tail of the queue
- //todo this polling will not work with multiple usernames but with single user
- // and multiple hosts, currently monitoring will work
- UserMonitorData take = null;
- JobStatusChangeRequestEvent jobStatus = new JobStatusChangeRequestEvent();
- MonitorID currentMonitorID = null;
- try {
- take = this.queue.take();
- List<HostMonitorData> hostMonitorData = take.getHostMonitorData();
- for (ListIterator<HostMonitorData> hostIterator = hostMonitorData.listIterator(); hostIterator.hasNext();) {
- HostMonitorData iHostMonitorData = hostIterator.next();
- if (iHostMonitorData.getJobSubmissionProtocol() == JobSubmissionProtocol.SSH) {
- String hostName = iHostMonitorData.getComputeResourceDescription().getHostName();
- ResourceConnection connection = null;
- if (connections.containsKey(hostName)) {
- if (!connections.get(hostName).isConnected()) {
- connection = new ResourceConnection(iHostMonitorData, getAuthenticationInfo());
- connections.put(hostName, connection);
- } else {
- logger.debug("We already have this connection so not going to create one");
- connection = connections.get(hostName);
- }
- } else {
- connection = new ResourceConnection(iHostMonitorData, getAuthenticationInfo());
- connections.put(hostName, connection);
- }
-
- // before we get the statuses, we check the cancel job list and remove them permanently
- List<MonitorID> monitorID = iHostMonitorData.getMonitorIDs();
- Iterator<String> iterator1 = cancelJobList.iterator();
- ListIterator<MonitorID> monitorIDListIterator = monitorID.listIterator();
- while (monitorIDListIterator.hasNext()) {
- MonitorID iMonitorID = monitorIDListIterator.next();
- while (iterator1.hasNext()) {
- String cancelMId = iterator1.next();
- if (cancelMId.equals(iMonitorID.getExperimentID() + "+" + iMonitorID.getTaskID())) {
- iMonitorID.setStatus(JobState.CANCELED);
-// CommonUtils.removeMonitorFromQueue(take, iMonitorID);
- removeList.add(iMonitorID);
- logger.debugId(cancelMId, "Found a match in cancel monitor queue, hence moved to the " +
- "completed job queue, experiment {}, task {} , job {}",
- iMonitorID.getExperimentID(), iMonitorID.getTaskID(), iMonitorID.getJobID());
- logger.info("Job cancelled: marking the Job as ************CANCELLED************ experiment {}, task {}, job name {} .",
- iMonitorID.getExperimentID(), iMonitorID.getTaskID(), iMonitorID.getJobName());
- sendNotification(iMonitorID);
- logger.info("To avoid timing issues we sleep sometime and try to retrieve output files");
- Thread.sleep(10000);
- GFacThreadPoolExecutor.getFixedThreadPool().submit(new OutHandlerWorker(gfac, iMonitorID, publisher));
- break;
- }
- }
- iterator1 = cancelJobList.iterator();
- }
-
- cleanup(take);
-
- synchronized (completedJobsFromPush) {
- for (ListIterator<String> iterator = completedJobsFromPush.listIterator(); iterator.hasNext(); ) {
- String completeId = iterator.next();
- for (monitorIDListIterator = monitorID.listIterator(); monitorIDListIterator.hasNext(); ) {
- MonitorID iMonitorID = monitorIDListIterator.next();
- if (completeId.equals(iMonitorID.getUserName() + "," + iMonitorID.getJobName())) {
- logger.info("This job is finished because push notification came with <username,jobName> " + completeId);
- iMonitorID.setStatus(JobState.COMPLETE);
-// CommonUtils.removeMonitorFromQueue(take, iMonitorID);//we have to make this empty everytime we iterate, otherwise this list will accumulate and will lead to a memory leak
- removeList.add(iMonitorID);
- logger.debugId(completeId, "Push notification updated job {} status to {}. " +
- "experiment {} , task {}.", iMonitorID.getJobID(), JobState.COMPLETE.toString(),
- iMonitorID.getExperimentID(), iMonitorID.getTaskID());
- logger.info("AMQP message recieved: marking the Job as ************COMPLETE************ experiment {}, task {}, job name {} .",
- iMonitorID.getExperimentID(), iMonitorID.getTaskID(), iMonitorID.getJobName());
-
- sendNotification(iMonitorID);
- GFacThreadPoolExecutor.getFixedThreadPool().submit(new OutHandlerWorker(gfac, iMonitorID, publisher));
- break;
- }
- }
- }
- }
-
- cleanup(take);
-
- // we have to get this again because we removed the already completed jobs with amqp messages
- monitorID = iHostMonitorData.getMonitorIDs();
- Map<String, JobState> jobStatuses = connection.getJobStatuses(monitorID);
- for (Iterator<MonitorID> iterator = monitorID.listIterator(); iterator.hasNext(); ) {
- MonitorID iMonitorID = iterator.next();
- currentMonitorID = iMonitorID;
- if (!JobState.CANCELED.equals(iMonitorID.getStatus()) &&
- !JobState.COMPLETE.equals(iMonitorID.getStatus())) {
- iMonitorID.setStatus(jobStatuses.get(iMonitorID.getJobID() + "," + iMonitorID.getJobName())); //IMPORTANT this is NOT a simple setter we have a logic
- } else if (JobState.COMPLETE.equals(iMonitorID.getStatus())) {
- logger.debugId(iMonitorID.getJobID(), "Moved job {} to completed jobs map, experiment {}, " +
- "task {}", iMonitorID.getJobID(), iMonitorID.getExperimentID(), iMonitorID.getTaskID());
-// CommonUtils.removeMonitorFromQueue(take, iMonitorID);
- removeList.add(iMonitorID);
- logger.info("PULL Notification is complete: marking the Job as ************COMPLETE************ experiment {}, task {}, job name {} .",
- iMonitorID.getExperimentID(), iMonitorID.getTaskID(), iMonitorID.getJobName());
- GFacThreadPoolExecutor.getFixedThreadPool().submit(new OutHandlerWorker(gfac, iMonitorID, publisher));
- }
- iMonitorID.setStatus(jobStatuses.get(iMonitorID.getJobID() + "," + iMonitorID.getJobName())); //IMPORTANT this is not a simple setter we have a logic
- iMonitorID.setLastMonitored(new Timestamp((new Date()).getTime()));
- sendNotification(iMonitorID);
- // if the job is completed we do not have to put the job to the queue again
- iMonitorID.setLastMonitored(new Timestamp((new Date()).getTime()));
- }
-
- cleanup(take);
-
-
- for (Iterator<MonitorID> iterator = monitorID.listIterator(); iterator.hasNext(); ) {
- MonitorID iMonitorID = iterator.next();
- if (iMonitorID.getFailedCount() > FAILED_COUNT) {
- iMonitorID.setLastMonitored(new Timestamp((new Date()).getTime()));
- String outputDir = iMonitorID.getJobExecutionContext().getOutputDir();
- List<String> stdOut = null;
- try {
- stdOut = connection.getCluster().listDirectory(outputDir); // check the outputs directory
- } catch (SSHApiException e) {
- if (e.getMessage().contains("No such file or directory")) {
- // this is because while we run output handler something failed and during exception
- // we store all the jobs in the monitor queue again
- logger.error("We know this job is already attempted to run out-handlers");
-// CommonUtils.removeMonitorFromQueue(queue, iMonitorID);
- }
- }
- if (stdOut != null && stdOut.size() > 0 && !stdOut.get(0).isEmpty()) { // have to be careful with this
- iMonitorID.setStatus(JobState.COMPLETE);
- logger.errorId(iMonitorID.getJobID(), "Job monitoring failed {} times, " +
- " Experiment {} , task {}", iMonitorID.getFailedCount(),
- iMonitorID.getExperimentID(), iMonitorID.getTaskID());
- logger.info("Listing directory came as complete: marking the Job as ************COMPLETE************ experiment {}, task {}, job name {} .",
- iMonitorID.getExperimentID(), iMonitorID.getTaskID(), iMonitorID.getJobName());
- sendNotification(iMonitorID);
-// CommonUtils.removeMonitorFromQueue(take, iMonitorID);
- removeList.add(iMonitorID);
- GFacThreadPoolExecutor.getFixedThreadPool().submit(new OutHandlerWorker(gfac, iMonitorID, publisher));
- } else {
- iMonitorID.setFailedCount(0);
- }
- } else {
- // Evey
- iMonitorID.setLastMonitored(new Timestamp((new Date()).getTime()));
- // if the job is complete we remove it from the Map, if any of these maps
- // get empty this userMonitorData will get delete from the queue
- }
- }
-
- cleanup(take);
-
-
- } else {
- logger.debug("Qstat Monitor doesn't handle non-gsissh hosts , host {}", iHostMonitorData.
- getComputeResourceDescription().getHostName());
- }
- }
- // We have finished all the HostMonitorData object in userMonitorData, now we need to put it back
- // now the userMonitorData goes back to the tail of the queue
- // during individual monitorID removal we remove the HostMonitorData object if it become empty
- // so if all the jobs are finished for all the hostMOnitorId objects in userMonitorData object
- // we should remove it from the queue so here we do not put it back.
- for (ListIterator<HostMonitorData> iterator1 = take.getHostMonitorData().listIterator(); iterator1.hasNext(); ) {
- HostMonitorData iHostMonitorID = iterator1.next();
- if (iHostMonitorID.getMonitorIDs().size() == 0) {
- iterator1.remove();
- logger.debug("Removed host {} from monitoring queue", iHostMonitorID.getComputeResourceDescription().getHostName());
- }
- }
- if(take.getHostMonitorData().size()!=0) {
- queue.put(take);
- }
- } catch (InterruptedException e) {
- if (!this.queue.contains(take)) {
- try {
- this.queue.put(take);
- } catch (InterruptedException e1) {
- e1.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
- }
- }
- logger.error("Error handling the job with Job ID:" + currentMonitorID.getJobID());
- throw new AiravataMonitorException(e);
- } catch (SSHApiException e) {
- logger.error(e.getMessage());
- if (e.getMessage().contains("Unknown Job Id Error")) {
- // in this case job is finished or may be the given job ID is wrong
- jobStatus.setState(JobState.UNKNOWN);
- JobIdentifier jobIdentifier = new JobIdentifier("UNKNOWN", "UNKNOWN", "UNKNOWN", "UNKNOWN", "UNKNOWN");
- if (currentMonitorID != null){
- jobIdentifier.setExperimentId(currentMonitorID.getExperimentID());
- jobIdentifier.setTaskId(currentMonitorID.getTaskID());
- jobIdentifier.setWorkflowNodeId(currentMonitorID.getWorkflowNodeID());
- jobIdentifier.setJobId(currentMonitorID.getJobID());
- jobIdentifier.setGatewayId(currentMonitorID.getJobExecutionContext().getGatewayID());
- }
- jobStatus.setJobIdentity(jobIdentifier);
- publisher.publish(jobStatus);
- } else if (e.getMessage().contains("illegally formed job identifier")) {
- logger.error("Wrong job ID is given so dropping the job from monitoring system");
- } else if (!this.queue.contains(take)) {
- try {
- queue.put(take);
- } catch (InterruptedException e1) {
- e1.printStackTrace();
- }
- }
- throw new AiravataMonitorException("Error retrieving the job status", e);
- } catch (Exception e) {
- try {
- queue.put(take);
- } catch (InterruptedException e1) {
- e1.printStackTrace();
- }
- throw new AiravataMonitorException("Error retrieving the job status", e);
- }
- return true;
- }
-
- private void sendNotification(MonitorID iMonitorID) {
- JobStatusChangeRequestEvent jobStatus = new JobStatusChangeRequestEvent();
- JobIdentifier jobIdentity = new JobIdentifier(iMonitorID.getJobID(),
- iMonitorID.getTaskID(),
- iMonitorID.getWorkflowNodeID(),
- iMonitorID.getExperimentID(),
- iMonitorID.getJobExecutionContext().getGatewayID());
- jobStatus.setJobIdentity(jobIdentity);
- jobStatus.setState(iMonitorID.getStatus());
- // we have this JobStatus class to handle amqp monitoring
- logger.debugId(jobStatus.getJobIdentity().getJobId(), "Published job status change request, " +
- "experiment {} , task {}", jobStatus.getJobIdentity().getExperimentId(),
- jobStatus.getJobIdentity().getTaskId());
-
- publisher.publish(jobStatus);
- }
-
- /**
- * This is the method to stop the polling process
- *
- * @return if the stopping process is successful return true else false
- */
- public boolean stopPulling() {
- this.startPulling = false;
- return true;
- }
-
- public MonitorPublisher getPublisher() {
- return publisher;
- }
-
- public void setPublisher(MonitorPublisher publisher) {
- this.publisher = publisher;
- }
-
- public BlockingQueue<UserMonitorData> getQueue() {
- return queue;
- }
-
- public void setQueue(BlockingQueue<UserMonitorData> queue) {
- this.queue = queue;
- }
-
- public boolean authenticate() {
- return false; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- public Map<String, ResourceConnection> getConnections() {
- return connections;
- }
-
- public boolean isStartPulling() {
- return startPulling;
- }
-
- public void setConnections(Map<String, ResourceConnection> connections) {
- this.connections = connections;
- }
-
- public void setStartPulling(boolean startPulling) {
- this.startPulling = startPulling;
- }
-
- public GFac getGfac() {
- return gfac;
- }
-
- public void setGfac(GFac gfac) {
- this.gfac = gfac;
- }
-
- public AuthenticationInfo getAuthenticationInfo() {
- return authenticationInfo;
- }
-
- public void setAuthenticationInfo(AuthenticationInfo authenticationInfo) {
- this.authenticationInfo = authenticationInfo;
- }
-
- public LinkedBlockingQueue<String> getCancelJobList() {
- return cancelJobList;
- }
-
- public void setCancelJobList(LinkedBlockingQueue<String> cancelJobList) {
- this.cancelJobList = cancelJobList;
- }
-
-
- private void cleanup(UserMonitorData userMonitorData){
- for(MonitorID iMonitorId:removeList){
- try {
- CommonUtils.removeMonitorFromQueue(userMonitorData, iMonitorId);
- } catch (AiravataMonitorException e) {
- logger.error(e.getMessage(), e);
- logger.error("Error deleting the monitor data: " + iMonitorId.getJobID());
- }
- }
- removeList.clear();
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java
deleted file mode 100644
index f718535..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.monitor.impl.pull.qstat;
-
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.SecurityContext;
-import org.apache.airavata.gfac.core.monitor.MonitorID;
-import org.apache.airavata.gfac.gsissh.security.GSISecurityContext;
-import org.apache.airavata.gfac.monitor.HostMonitorData;
-import org.apache.airavata.gfac.ssh.security.SSHSecurityContext;
-import org.apache.airavata.gsi.ssh.api.SSHApiException;
-import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
-import org.apache.airavata.gsi.ssh.impl.JobStatus;
-import org.apache.airavata.gsi.ssh.impl.PBSCluster;
-import org.apache.airavata.model.workspace.experiment.JobState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-
-public class ResourceConnection {
- private static final Logger log = LoggerFactory.getLogger(ResourceConnection.class);
-
- private PBSCluster cluster;
-
- private AuthenticationInfo authenticationInfo;
-
-
- public ResourceConnection(HostMonitorData hostMonitorData,AuthenticationInfo authInfo) throws SSHApiException {
- MonitorID monitorID = hostMonitorData.getMonitorIDs().get(0);
- try {
- SecurityContext securityContext = monitorID.getJobExecutionContext().getSecurityContext(monitorID.getComputeResourceDescription().getHostName());
- if(securityContext != null) {
- if (securityContext instanceof GSISecurityContext) {
- GSISecurityContext gsiSecurityContext = (GSISecurityContext) securityContext;
- cluster = (PBSCluster) gsiSecurityContext.getPbsCluster();
- } else if (securityContext instanceof SSHSecurityContext) {
- SSHSecurityContext sshSecurityContext = (SSHSecurityContext)
- securityContext;
- cluster = (PBSCluster) sshSecurityContext.getPbsCluster();
- }
- }
- // we just use cluster configuration from the incoming request and construct a new cluster because for monitoring
- // we are using our own credentials and not using one users account to do everything.
- authenticationInfo = authInfo;
- } catch (GFacException e) {
- log.error("Error reading data from job ExecutionContext");
- }
- }
-
- public ResourceConnection(HostMonitorData hostMonitorData) throws SSHApiException {
- MonitorID monitorID = hostMonitorData.getMonitorIDs().get(0);
- try {
- GSISecurityContext securityContext = (GSISecurityContext)
- monitorID.getJobExecutionContext().getSecurityContext(monitorID.getComputeResourceDescription().getHostName());
- cluster = (PBSCluster) securityContext.getPbsCluster();
-
- // we just use cluster configuration from the incoming request and construct a new cluster because for monitoring
- // we are using our own credentials and not using one users account to do everything.
- cluster = new PBSCluster(cluster.getServerInfo(), authenticationInfo, cluster.getJobManagerConfiguration());
- } catch (GFacException e) {
- log.error("Error reading data from job ExecutionContext");
- }
- }
-
- public JobState getJobStatus(MonitorID monitorID) throws SSHApiException {
- String jobID = monitorID.getJobID();
- //todo so currently we execute the qstat for each job but we can use user based monitoring
- //todo or we should concatenate all the commands and execute them in one go and parseSingleJob the response
- return getStatusFromString(cluster.getJobStatus(jobID).toString());
- }
-
- public Map<String, JobState> getJobStatuses(List<MonitorID> monitorIDs) throws SSHApiException {
- Map<String, JobStatus> treeMap = new TreeMap<String, JobStatus>();
- Map<String, JobState> treeMap1 = new TreeMap<String, JobState>();
- // creating a sorted map with all the jobIds and with the predefined
- // status as UNKNOWN
- for (MonitorID monitorID : monitorIDs) {
- treeMap.put(monitorID.getJobID()+","+monitorID.getJobName(), JobStatus.U);
- }
- String userName = cluster.getServerInfo().getUserName();
- //todo so currently we execute the qstat for each job but we can use user based monitoring
- //todo or we should concatenate all the commands and execute them in one go and parseSingleJob the response
- //
- cluster.getJobStatuses(userName, treeMap);
- for (String key : treeMap.keySet()) {
- treeMap1.put(key, getStatusFromString(treeMap.get(key).toString()));
- }
- return treeMap1;
- }
-
- private JobState getStatusFromString(String status) {
- log.info("parsing the job status returned : " + status);
- if (status != null) {
- if ("C".equals(status) || "CD".equals(status) || "E".equals(status) || "CG".equals(status) || "DONE".equals(status)) {
- return JobState.COMPLETE;
- } else if ("H".equals(status) || "h".equals(status)) {
- return JobState.HELD;
- } else if ("Q".equals(status) || "qw".equals(status) || "PEND".equals(status)) {
- return JobState.QUEUED;
- } else if ("R".equals(status) || "CF".equals(status) || "r".equals(status) || "RUN".equals(status)) {
- return JobState.ACTIVE;
- } else if ("T".equals(status)) {
- return JobState.HELD;
- } else if ("W".equals(status) || "PD".equals(status)) {
- return JobState.QUEUED;
- } else if ("S".equals(status) || "PSUSP".equals(status) || "USUSP".equals(status) || "SSUSP".equals(status)) {
- return JobState.SUSPENDED;
- } else if ("CA".equals(status)) {
- return JobState.CANCELED;
- } else if ("F".equals(status) || "NF".equals(status) || "TO".equals(status) || "EXIT".equals(status)) {
- return JobState.FAILED;
- } else if ("PR".equals(status) || "Er".equals(status)) {
- return JobState.FAILED;
- } else if ("U".equals(status) || ("UNKWN".equals(status))) {
- return JobState.UNKNOWN;
- }
- }
- return JobState.UNKNOWN;
- }
-
- public PBSCluster getCluster() {
- return cluster;
- }
-
- public void setCluster(PBSCluster cluster) {
- this.cluster = cluster;
- }
-
- public boolean isConnected(){
- return this.cluster.getSession().isConnected();
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/AMQPMonitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/AMQPMonitor.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/AMQPMonitor.java
deleted file mode 100644
index de8cd8c..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/AMQPMonitor.java
+++ /dev/null
@@ -1,280 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.monitor.impl.push.amqp;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-
-import org.apache.airavata.common.utils.MonitorPublisher;
-import org.apache.airavata.common.utils.ServerSettings;
-import org.apache.airavata.gfac.core.monitor.MonitorID;
-import org.apache.airavata.gfac.monitor.core.PushMonitor;
-import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
-import org.apache.airavata.gfac.monitor.util.AMQPConnectionUtil;
-import org.apache.airavata.gfac.monitor.util.CommonUtils;
-import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
-import org.apache.airavata.model.messaging.event.JobIdentifier;
-import org.apache.airavata.model.messaging.event.JobStatusChangeEvent;
-import org.apache.airavata.model.workspace.experiment.JobState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.eventbus.EventBus;
-import com.google.common.eventbus.Subscribe;
-import com.rabbitmq.client.Channel;
-import com.rabbitmq.client.Connection;
-
-/**
- * This is the implementation for AMQP based finishQueue, this uses
- * rabbitmq client to recieve AMQP based monitoring data from
- * mostly excede resources.
- */
-public class AMQPMonitor extends PushMonitor {
- private final static Logger logger = LoggerFactory.getLogger(AMQPMonitor.class);
-
-
- /* this will keep all the channels available in the system, we do not create
- channels for all the jobs submitted, but we create channels for each user for each
- host.
- */
- private Map<String, Channel> availableChannels;
-
- private MonitorPublisher publisher;
-
- private MonitorPublisher localPublisher;
-
- private BlockingQueue<MonitorID> runningQueue;
-
- private BlockingQueue<MonitorID> finishQueue;
-
- private String connectionName;
-
- private String proxyPath;
-
- private List<String> amqpHosts;
-
- private boolean startRegister;
-
- public AMQPMonitor(){
-
- }
- public AMQPMonitor(MonitorPublisher publisher, BlockingQueue<MonitorID> runningQueue,
- BlockingQueue<MonitorID> finishQueue,
- String proxyPath,String connectionName,List<String> hosts) {
- this.publisher = publisher;
- this.runningQueue = runningQueue; // these will be initialized by the MonitorManager
- this.finishQueue = finishQueue; // these will be initialized by the MonitorManager
- this.availableChannels = new HashMap<String, Channel>();
- this.connectionName = connectionName;
- this.proxyPath = proxyPath;
- this.amqpHosts = hosts;
- this.localPublisher = new MonitorPublisher(new EventBus());
- this.localPublisher.registerListener(this);
- }
-
- public void initialize(String proxyPath, String connectionName, List<String> hosts) {
- this.availableChannels = new HashMap<String, Channel>();
- this.connectionName = connectionName;
- this.proxyPath = proxyPath;
- this.amqpHosts = hosts;
- this.localPublisher = new MonitorPublisher(new EventBus());
- this.localPublisher.registerListener(this);
- }
-
- @Override
- public boolean registerListener(MonitorID monitorID) throws AiravataMonitorException {
- // we subscribe to read user-host based subscription
- ComputeResourceDescription computeResourceDescription = monitorID.getComputeResourceDescription();
- if (computeResourceDescription.isSetIpAddresses() && computeResourceDescription.getIpAddresses().size() > 0) {
- // we get first ip address for the moment
- String hostAddress = computeResourceDescription.getIpAddresses().get(0);
- // in amqp case there are no multiple jobs per each host, because once a job is put in to the queue it
- // will be picked by the Monitor, so jobs will not stay in this queueu but jobs will stay in finishQueue
- String channelID = CommonUtils.getChannelID(monitorID);
- if (availableChannels.get(channelID) == null) {
- try {
- //todo need to fix this rather getting it from a file
- Connection connection = AMQPConnectionUtil.connect(amqpHosts, connectionName, proxyPath);
- Channel channel = null;
- channel = connection.createChannel();
- availableChannels.put(channelID, channel);
- String queueName = channel.queueDeclare().getQueue();
-
- BasicConsumer consumer = new
- BasicConsumer(new JSONMessageParser(), localPublisher); // here we use local publisher
- channel.basicConsume(queueName, true, consumer);
- String filterString = CommonUtils.getRoutingKey(monitorID.getUserName(), hostAddress);
- // here we queuebind to a particular user in a particular machine
- channel.queueBind(queueName, "glue2.computing_activity", filterString);
- logger.info("Using filtering string to monitor: " + filterString);
- } catch (IOException e) {
- logger.error("Error creating the connection to finishQueue the job:" + monitorID.getUserName());
- }
- }
- } else {
- throw new AiravataMonitorException("Couldn't register monitor for jobId :" + monitorID.getJobID() +
- " , ComputeResourceDescription " + computeResourceDescription.getHostName() + " doesn't has an " +
- "IpAddress with it");
- }
- return true;
- }
-
- public void run() {
- // before going to the while true mode we start unregister thread
- startRegister = true; // this will be unset by someone else
- while (startRegister || !ServerSettings.isStopAllThreads()) {
- try {
- MonitorID take = runningQueue.take();
- this.registerListener(take);
- } catch (AiravataMonitorException e) { // catch any exceptino inside the loop
- logger.error(e.getMessage(), e);
- } catch (InterruptedException e) {
- logger.error(e.getMessage(), e);
- } catch (Exception e){
- logger.error(e.getMessage(), e);
- }
- }
- Set<String> strings = availableChannels.keySet();
- for(String key:strings) {
- Channel channel = availableChannels.get(key);
- try {
- channel.close();
- } catch (IOException e) {
- logger.error(e.getMessage(), e);
- }
- }
- }
-
- @Subscribe
- public boolean unRegisterListener(MonitorID monitorID) throws AiravataMonitorException {
- Iterator<MonitorID> iterator = finishQueue.iterator();
- MonitorID next = null;
- while(iterator.hasNext()){
- next = iterator.next();
- if(next.getJobID().endsWith(monitorID.getJobID())){
- break;
- }
- }
- if(next == null) {
- logger.error("Job has removed from the queue, old obsolete message recieved");
- return false;
- }
- String channelID = CommonUtils.getChannelID(next);
- if (JobState.FAILED.equals(monitorID.getStatus()) || JobState.COMPLETE.equals(monitorID.getStatus())) {
- finishQueue.remove(next);
-
- // if this is the last job in the queue at this point with the same username and same host we
- // close the channel and close the connection and remove it from availableChannels
- if (CommonUtils.isTheLastJobInQueue(finishQueue, next)) {
- logger.info("There are no jobs to monitor for common ChannelID:" + channelID + " , so we unsubscribe it" +
- ", incase new job created we do subscribe again");
- Channel channel = availableChannels.get(channelID);
- if (channel == null) {
- logger.error("Already Unregistered the listener");
- throw new AiravataMonitorException("Already Unregistered the listener");
- } else {
- try {
- channel.queueUnbind(channel.queueDeclare().getQueue(), "glue2.computing_activity", CommonUtils.getRoutingKey(next));
- channel.close();
- channel.getConnection().close();
- availableChannels.remove(channelID);
- } catch (IOException e) {
- logger.error("Error unregistering the listener");
- throw new AiravataMonitorException("Error unregistering the listener");
- }
- }
- }
- }
- next.setStatus(monitorID.getStatus());
- JobIdentifier jobIdentity = new JobIdentifier(next.getJobID(),
- next.getTaskID(),
- next.getWorkflowNodeID(),
- next.getExperimentID(),
- next.getJobExecutionContext().getGatewayID());
- publisher.publish(new JobStatusChangeEvent(next.getStatus(),jobIdentity));
- return true;
- }
- @Override
- public boolean stopRegister() throws AiravataMonitorException {
- return false; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- public Map<String, Channel> getAvailableChannels() {
- return availableChannels;
- }
-
- public void setAvailableChannels(Map<String, Channel> availableChannels) {
- this.availableChannels = availableChannels;
- }
-
- public MonitorPublisher getPublisher() {
- return publisher;
- }
-
- public void setPublisher(MonitorPublisher publisher) {
- this.publisher = publisher;
- }
-
- public BlockingQueue<MonitorID> getRunningQueue() {
- return runningQueue;
- }
-
- public void setRunningQueue(BlockingQueue<MonitorID> runningQueue) {
- this.runningQueue = runningQueue;
- }
-
- public BlockingQueue<MonitorID> getFinishQueue() {
- return finishQueue;
- }
-
- public void setFinishQueue(BlockingQueue<MonitorID> finishQueue) {
- this.finishQueue = finishQueue;
- }
-
- public String getProxyPath() {
- return proxyPath;
- }
-
- public void setProxyPath(String proxyPath) {
- this.proxyPath = proxyPath;
- }
-
- public List<String> getAmqpHosts() {
- return amqpHosts;
- }
-
- public void setAmqpHosts(List<String> amqpHosts) {
- this.amqpHosts = amqpHosts;
- }
-
- public boolean isStartRegister() {
- return startRegister;
- }
-
- public void setStartRegister(boolean startRegister) {
- this.startRegister = startRegister;
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/BasicConsumer.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/BasicConsumer.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/BasicConsumer.java
deleted file mode 100644
index bd5c625..0000000
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/BasicConsumer.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.monitor.impl.push.amqp;
-
-import org.apache.airavata.common.utils.MonitorPublisher;
-import org.apache.airavata.gfac.core.monitor.MonitorID;
-import org.apache.airavata.gfac.monitor.core.MessageParser;
-import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.rabbitmq.client.AMQP;
-import com.rabbitmq.client.Consumer;
-import com.rabbitmq.client.Envelope;
-import com.rabbitmq.client.ShutdownSignalException;
-
-public class BasicConsumer implements Consumer {
- private final static Logger logger = LoggerFactory.getLogger(AMQPMonitor.class);
-
- private MessageParser parser;
-
- private MonitorPublisher publisher;
-
- public BasicConsumer(MessageParser parser, MonitorPublisher publisher) {
- this.parser = parser;
- this.publisher = publisher;
- }
-
- public void handleCancel(String consumerTag) {
- }
-
- public void handleCancelOk(String consumerTag) {
- }
-
- public void handleConsumeOk(String consumerTag) {
- }
-
- public void handleDelivery(String consumerTag,
- Envelope envelope,
- AMQP.BasicProperties properties,
- byte[] body) {
-
- logger.debug("job update for: " + envelope.getRoutingKey());
- String message = new String(body);
- message = message.replaceAll("(?m)^", " ");
- // Here we parse the message and get the job status and push it
- // to the Event bus, this will be picked by
-// AiravataJobStatusUpdator and store in to registry
-
- logger.debug("************************************************************");
- logger.debug("AMQP Message recieved \n" + message);
- logger.debug("************************************************************");
- try {
- String jobID = envelope.getRoutingKey().split("\\.")[0];
- MonitorID monitorID = new MonitorID(null, jobID, null, null, null, null,null);
- monitorID.setStatus(parser.parseMessage(message));
- publisher.publish(monitorID);
- } catch (AiravataMonitorException e) {
- logger.error(e.getMessage(), e);
- }
- }
-
- public void handleRecoverOk(String consumerTag) {
- }
-
- public void handleShutdownSignal(String consumerTag, ShutdownSignalException sig) {
- }
-
-}