You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by sh...@apache.org on 2016/03/25 19:11:35 UTC

[1/2] airavata git commit: Make unicore works with local files

Repository: airavata
Updated Branches:
  refs/heads/develop fd9b64feb -> 253dee8cd


Make unicore works with local files


Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/ef4329c5
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/ef4329c5
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/ef4329c5

Branch: refs/heads/develop
Commit: ef4329c51c9d81c41c42c5dd77eb5ca34ef40675
Parents: 6c483bc
Author: Shameera Rathnayaka <sh...@gmail.com>
Authored: Fri Mar 25 14:11:07 2016 -0400
Committer: Shameera Rathnayaka <sh...@gmail.com>
Committed: Fri Mar 25 14:11:07 2016 -0400

----------------------------------------------------------------------
 .../gfac/impl/task/BESJobSubmissionTask.java    | 48 ++++++++------
 .../impl/task/utils/bes/DataTransferrer.java    | 70 +++++++++++---------
 2 files changed, 64 insertions(+), 54 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/airavata/blob/ef4329c5/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java
index 145cde9..678924f 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java
@@ -77,8 +77,9 @@ public class BESJobSubmissionTask implements JobSubmissionTask {
     public TaskStatus execute(TaskContext taskContext) {
         TaskStatus taskStatus = new TaskStatus(TaskState.CREATED);
         StorageClient sc = null;
+        // FIXME - use original output dir
+        taskContext.getParentProcessContext().setOutputDir("");
 
-        //TODO - initialize securityContext secProperties
         try {
             if (secProperties == null) {
                 secProperties = getSecurityConfig(taskContext.getParentProcessContext());
@@ -121,6 +122,8 @@ public class BESJobSubmissionTask implements JobSubmissionTask {
             dt.uploadLocalFiles();
 
             JobModel jobDetails = new JobModel();
+            jobDetails.setTaskId(taskContext.getTaskId());
+            jobDetails.setProcessId(taskContext.getProcessId());
             FactoryClient factory = new FactoryClient(eprt, secProperties);
 
             log.info(String.format("Activity Submitting to %s ... \n",
@@ -143,6 +146,7 @@ public class BESJobSubmissionTask implements JobSubmissionTask {
             jobDetails.setJobDescription(activityEpr.toString());
             jobDetails.setJobStatus(new JobStatus(JobState.SUBMITTED));
             processContext.setJobModel(jobDetails);
+            GFacUtils.saveJobModel(processContext, jobDetails);
             GFacUtils.saveJobStatus(processContext, jobDetails);
             log.info(formatStatusMessage(activityEpr.getAddress()
                     .getStringValue(), factory.getActivityStatus(activityEpr)
@@ -164,7 +168,7 @@ public class BESJobSubmissionTask implements JobSubmissionTask {
                         + "\n"
                         + activityStatus.getFault().getFaultstring()
                         + "\n EXITCODE: " + activityStatus.getExitCode();
-                log.info(error);
+                log.error(error);
 
                 JobState applicationJobStatus = JobState.FAILED;
                 jobDetails.setJobStatus(new JobStatus(applicationJobStatus));
@@ -173,7 +177,7 @@ public class BESJobSubmissionTask implements JobSubmissionTask {
 
                 //What if job is failed before execution and there are not stdouts generated yet?
                 log.debug("Downloading any standard output and error files, if they were produced.");
-                dt.downloadStdOuts();
+                dt.downloadRemoteFiles();
 
             } else if (activityStatus.getState() == ActivityStateEnumeration.CANCELLED) {
                 JobState applicationJobStatus = JobState.CANCELED;
@@ -184,17 +188,19 @@ public class BESJobSubmissionTask implements JobSubmissionTask {
             } else if (activityStatus.getState() == ActivityStateEnumeration.FINISHED) {
                 try {
                     Thread.sleep(5000);
-                    JobState applicationJobStatus = JobState.COMPLETE;
-                    jobDetails.setJobStatus(new JobStatus(applicationJobStatus));
-                    GFacUtils.saveJobStatus(processContext, jobDetails);
-
-                } catch (InterruptedException e) {
-                }
-                if (activityStatus.getExitCode() == 0) {
-                    dt.downloadRemoteFiles();
-                } else {
-                    dt.downloadStdOuts();
+                } catch (InterruptedException ignored) {
                 }
+                JobState applicationJobStatus = JobState.COMPLETE;
+                jobDetails.setJobStatus(new JobStatus(applicationJobStatus));
+                GFacUtils.saveJobStatus(processContext, jobDetails);
+                log.info("Job Id: {}, exit code: {}, exit status: {}", jobDetails.getJobId(),
+                        activityStatus.getExitCode(), ActivityStateEnumeration.FINISHED.toString());
+
+//                if (activityStatus.getExitCode() == 0) {
+//                } else {
+//                    dt.downloadStdOuts();
+//                }
+                dt.downloadRemoteFiles();
             }
 
             dt.publishFinalOutputs();
@@ -243,22 +249,22 @@ public class BESJobSubmissionTask implements JobSubmissionTask {
             FactoryClient factoryClient = new FactoryClient(factoryEpr, secProperties);
             JobState applicationJobStatus = null;
 
-            while ((factoryClient.getActivityStatus(activityEpr) != ActivityStateEnumeration.FINISHED)
-                    && (factoryClient.getActivityStatus(activityEpr) != ActivityStateEnumeration.FAILED)
-                    && (factoryClient.getActivityStatus(activityEpr) != ActivityStateEnumeration.CANCELLED)
+            ActivityStateEnumeration.Enum activityStatus = factoryClient.getActivityStatus(activityEpr);
+            while ((activityStatus != ActivityStateEnumeration.FINISHED)
+                    && (activityStatus != ActivityStateEnumeration.FAILED)
+                    && (activityStatus != ActivityStateEnumeration.CANCELLED)
                     && (applicationJobStatus != JobState.COMPLETE)) {
 
-                ActivityStatusType activityStatus = getStatus(factoryClient, activityEpr);
-                applicationJobStatus = getApplicationJobStatus(activityStatus);
-
+                ActivityStatusType activityStatusType = getStatus(factoryClient, activityEpr);
+                applicationJobStatus = getApplicationJobStatus(activityStatusType);
                 sendNotification(processContext,processContext.getJobModel());
-
                 // GFacUtils.updateApplicationJobStatus(jobExecutionContext,jobId,
                 // applicationJobStatus);
                 try {
                     Thread.sleep(5000);
                 } catch (InterruptedException e) {}
-                continue;
+
+                activityStatus = factoryClient.getActivityStatus(activityEpr);
             }
         } catch(Exception e) {
             log.error("Error monitoring job status..");

http://git-wip-us.apache.org/repos/asf/airavata/blob/ef4329c5/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/DataTransferrer.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/DataTransferrer.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/DataTransferrer.java
index 4a0cbbf..b4bf9ed 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/DataTransferrer.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/DataTransferrer.java
@@ -150,25 +150,32 @@ public class DataTransferrer {
         String stderrFileName = new File(stderrLocation).getName();
 
         FileDownloader f1 = null;
-        try {
-            log.info("Downloading stdout and stderr..");
-            log.info(stdoutFileName + " -> "+stdoutLocation);
+        log.info("Downloading stdout and stderr..");
+        log.info(stdoutFileName + " -> " + stdoutLocation);
 
-            f1 = new FileDownloader(stdoutFileName,stdoutLocation, Mode.overwrite);
+        f1 = new FileDownloader(stdoutFileName, stdoutLocation, Mode.overwrite);
+        try {
             f1.perform(storageClient);
-            String stdoutput = readFile(stdoutLocation);
+//            String stdoutput = readFile(stdoutLocation);
+        } catch (Exception e) {
+            log.error("Error while downloading " + stdoutFileName + " to location " + stdoutLocation, e);
+        }
 
-            log.info(stderrFileName + " -> " + stderrLocation);
-            f1.setFrom(stderrFileName);
-            f1.setTo(stderrLocation);
+        log.info(stderrFileName + " -> " + stderrLocation);
+        f1.setFrom(stderrFileName);
+        f1.setTo(stderrLocation);
+        try {
             f1.perform(storageClient);
-            String stderror = readFile(stderrLocation);
-
-            if(UASDataStagingProcessor.isUnicoreEndpoint(processContext)) {
-                String scriptExitCodeFName = "UNICORE_SCRIPT_EXIT_CODE";
-                String scriptCodeLocation = gatewayDownloadLocation+File.separator+scriptExitCodeFName;
-                f1.setFrom(scriptExitCodeFName);
-                f1.setTo(scriptCodeLocation);
+//            String stderror = readFile(stderrLocation);
+        } catch (Exception e) {
+            log.error("Error while downloading " + stderrFileName + " to location " + stderrLocation);
+        }
+        String scriptExitCodeFName = "UNICORE_SCRIPT_EXIT_CODE";
+        String scriptCodeLocation = gatewayDownloadLocation + File.separator + scriptExitCodeFName;
+        if (UASDataStagingProcessor.isUnicoreEndpoint(processContext)) {
+            f1.setFrom(scriptExitCodeFName);
+            f1.setTo(scriptCodeLocation);
+            try {
                 f1.perform(storageClient);
                 OutputDataObjectType output = new OutputDataObjectType();
                 output.setName(scriptExitCodeFName);
@@ -176,13 +183,12 @@ public class DataTransferrer {
                 output.setType(DataType.URI);
                 output.setIsRequired(true);
                 processContext.getProcessModel().getProcessOutputs().add(output);
-                log.info("UNICORE_SCRIPT_EXIT_CODE -> "+scriptCodeLocation);
-                log.info("EXIT CODE: "+ readFile(scriptCodeLocation));
+                log.info("UNICORE_SCRIPT_EXIT_CODE -> " + scriptCodeLocation);
+                log.info("EXIT CODE: " + readFile(scriptCodeLocation));
+            } catch (Exception e) {
+                log.error("Error downloading file " + scriptExitCodeFName + " to location " + scriptCodeLocation, e);
             }
-        } catch (Exception e) {
-            throw new GFacException(e.getLocalizedMessage(),e);
         }
-
     }
 
     private String readFile(String localFile) throws IOException {
@@ -253,7 +259,7 @@ public class DataTransferrer {
     public void downloadRemoteFiles() throws GFacException {
 
         if(log.isDebugEnabled()) {
-            log.debug("Download location is:"+gatewayDownloadLocation);
+            log.debug("Download location is:" + gatewayDownloadLocation);
         }
 
         List<OutputDataObjectType> applicationOutputs = processContext.getProcessModel().getProcessOutputs();
@@ -265,35 +271,33 @@ public class DataTransferrer {
                 if(output.getType().equals(DataType.STDOUT)) {
                     output.setValue(processContext.getStdoutLocation());
                     resultantOutputsLst.add(output);
-                }
-
-                else if(output.getType().equals(DataType.STDERR)) {
+                } else if(output.getType().equals(DataType.STDERR)) {
                     output.setValue(processContext.getStderrLocation());
                     resultantOutputsLst.add(output);
-                }
-                else if(output.getType().equals(DataType.STRING)) {
+                } else if (output.getType().equals(DataType.STRING)) {
                     String value = null;
-                    if(!output.getLocation().isEmpty()){
+                    if (!output.getLocation().isEmpty()) {
                         value = output.getLocation() + File.separator + output.getValue();
-                    }else{
+                    } else {
                         value = output.getValue();
                     }
                     String outputPath = gatewayDownloadLocation + File.separator + output.getValue();
                     File f = new File(gatewayDownloadLocation);
-                    if(!f.exists())
+                    if (!f.exists())
                         f.mkdirs();
 
-                    FileDownloader fileDownloader = new FileDownloader(value,outputPath, Mode.overwrite);
+                    FileDownloader fileDownloader = new FileDownloader(value, outputPath, Mode.overwrite);
                     try {
                         fileDownloader.perform(storageClient);
                         output.setType(DataType.URI);
                         output.setValue(outputPath);
-                        processContext.getProcessModel().getProcessOutputs().add(output);
                         resultantOutputsLst.add(output);
                     } catch (Exception e) {
-                        log.error("Error downloading "+value+" from job working directory. ");
-                        throw new GFacException(e.getLocalizedMessage(),e);
+                        log.error("Error downloading " + value + " from job working directory. ");
+//                        throw new GFacException(e.getLocalizedMessage(),e);
                     }
+                } else {
+                    log.info("Ignore output file {}, type {}", output.getValue(), output.getType().toString());
                 }
 
             }


[2/2] airavata git commit: Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/airavata into develop

Posted by sh...@apache.org.
Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/airavata into develop


Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/253dee8c
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/253dee8c
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/253dee8c

Branch: refs/heads/develop
Commit: 253dee8cd8771966b4ae9d8b323e1d9207f61414
Parents: ef4329c fd9b64f
Author: Shameera Rathnayaka <sh...@gmail.com>
Authored: Fri Mar 25 14:11:17 2016 -0400
Committer: Shameera Rathnayaka <sh...@gmail.com>
Committed: Fri Mar 25 14:11:17 2016 -0400

----------------------------------------------------------------------
 .../server/handler/AiravataServerHandler.java   |    55 +-
 .../api/server/handler/utils/DataCatInit.java   |   315 -
 .../server/handler/utils/ReplicaCatInit.java    |   315 +
 .../java/org/apache/airavata/api/Airavata.java  | 40373 +++++++++--------
 .../main/resources/lib/airavata/Airavata.cpp    |  3498 +-
 .../src/main/resources/lib/airavata/Airavata.h  |   378 +-
 .../lib/airavata/Airavata_server.skeleton.cpp   |    18 +-
 .../resources/lib/airavata/airavata_api_types.h |     2 +-
 .../lib/airavata/airavata_data_models_types.h   |     2 +-
 .../airavata/application_io_models_types.cpp    |    44 +
 .../lib/airavata/application_io_models_types.h  |    24 +-
 .../replica_catalog_models_constants.cpp        |     2 +-
 .../airavata/replica_catalog_models_constants.h |     2 +-
 .../airavata/replica_catalog_models_types.cpp   |   434 +-
 .../lib/airavata/replica_catalog_models_types.h |   143 +-
 .../resources/lib/Airavata/API/Airavata.php     | 11205 ++---
 .../lib/Airavata/Model/Application/Io/Types.php |    46 +
 .../lib/Airavata/Model/Data/Replica/Types.php   |   803 +
 .../lib/apache/airavata/api/Airavata-remote     |    14 +
 .../lib/apache/airavata/api/Airavata.py         |   594 +-
 .../resources/lib/apache/airavata/api/ttypes.py |     2 +-
 .../airavata/model/application/io/ttypes.py     |    30 +-
 .../airavata/model/data/replica/__init__.py     |     1 +
 .../airavata/model/data/replica/constants.py    |    11 +
 .../airavata/model/data/replica/ttypes.py       |   511 +
 .../lib/apache/airavata/model/ttypes.py         |     2 +-
 .../client/samples/DataCatalogSample.java       |    73 -
 .../client/samples/ReplicaCatalogSample.java    |    72 +
 .../apache/airavata/model/ComponentStatus.java  |     2 +-
 .../org/apache/airavata/model/EdgeModel.java    |     2 +-
 .../org/apache/airavata/model/NodeModel.java    |     2 +-
 .../org/apache/airavata/model/PortModel.java    |     2 +-
 .../apache/airavata/model/WorkflowModel.java    |     2 +-
 .../apache/airavata/model/WorkflowStatus.java   |     2 +-
 .../ApplicationDeploymentDescription.java       |     2 +-
 .../appdeployment/ApplicationModule.java        |     2 +-
 .../appcatalog/appdeployment/CommandObject.java |     2 +-
 .../appcatalog/appdeployment/SetEnvPaths.java   |     2 +-
 .../ApplicationInterfaceDescription.java        |     2 +-
 .../appcatalog/computeresource/BatchQueue.java  |     2 +-
 .../computeresource/CloudJobSubmission.java     |     2 +-
 .../ComputeResourceDescription.java             |     2 +-
 .../computeresource/GlobusJobSubmission.java    |     2 +-
 .../computeresource/JobSubmissionInterface.java |     2 +-
 .../computeresource/LOCALSubmission.java        |     2 +-
 .../computeresource/ResourceJobManager.java     |     2 +-
 .../computeresource/SSHJobSubmission.java       |     2 +-
 .../computeresource/UnicoreJobSubmission.java   |     2 +-
 .../ComputeResourcePreference.java              |     2 +-
 .../gatewayprofile/GatewayResourceProfile.java  |     2 +-
 .../gatewayprofile/StoragePreference.java       |     2 +-
 .../StorageResourceDescription.java             |     2 +-
 .../application/io/InputDataObjectType.java     |   116 +-
 .../application/io/OutputDataObjectType.java    |   116 +-
 .../airavata/model/commons/ErrorModel.java      |     2 +-
 .../model/commons/ValidationResults.java        |     2 +-
 .../airavata/model/commons/ValidatorResult.java |     2 +-
 .../data/movement/DataMovementInterface.java    |     2 +-
 .../data/movement/GridFTPDataMovement.java      |     2 +-
 .../model/data/movement/LOCALDataMovement.java  |     2 +-
 .../model/data/movement/SCPDataMovement.java    |     2 +-
 .../data/movement/UnicoreDataMovement.java      |     2 +-
 .../model/data/product/DataProductModel.java    |  1943 -
 .../model/data/product/DataProductType.java     |    65 -
 .../data/product/DataReplicaLocationModel.java  |  1635 -
 .../data/product/ReplicaLocationCategory.java   |    68 -
 .../data/product/ReplicaPersistentType.java     |    62 -
 .../model/data/replica/DataProductModel.java    |  1677 +
 .../model/data/replica/DataProductType.java     |    62 +
 .../data/replica/DataReplicaLocationModel.java  |  1635 +
 .../data/replica/ReplicaLocationCategory.java   |    68 +
 .../data/replica/ReplicaPersistentType.java     |    62 +
 .../model/error/AiravataClientException.java    |     2 +-
 .../model/error/AiravataSystemException.java    |     2 +-
 .../model/error/AuthenticationException.java    |     2 +-
 .../model/error/AuthorizationException.java     |     2 +-
 .../error/ExperimentNotFoundException.java      |     2 +-
 .../model/error/InvalidRequestException.java    |     2 +-
 .../model/error/LaunchValidationException.java  |     2 +-
 .../model/error/ProjectNotFoundException.java   |     2 +-
 .../airavata/model/error/TimedOutException.java |     2 +-
 .../airavata/model/error/ValidationResults.java |     2 +-
 .../airavata/model/error/ValidatorResult.java   |     2 +-
 .../model/experiment/ExperimentModel.java       |     2 +-
 .../model/experiment/ExperimentStatistics.java  |     2 +-
 .../experiment/ExperimentSummaryModel.java      |     2 +-
 .../experiment/UserConfigurationDataModel.java  |     2 +-
 .../org/apache/airavata/model/job/JobModel.java |     2 +-
 .../event/ExperimentStatusChangeEvent.java      |     2 +-
 .../model/messaging/event/JobIdentifier.java    |     2 +-
 .../messaging/event/JobStatusChangeEvent.java   |     2 +-
 .../event/JobStatusChangeRequestEvent.java      |     2 +-
 .../airavata/model/messaging/event/Message.java |     2 +-
 .../messaging/event/ProcessIdentifier.java      |     2 +-
 .../event/ProcessStatusChangeEvent.java         |     2 +-
 .../event/ProcessStatusChangeRequestEvent.java  |     2 +-
 .../messaging/event/ProcessSubmitEvent.java     |     2 +-
 .../messaging/event/ProcessTerminateEvent.java  |     2 +-
 .../model/messaging/event/TaskIdentifier.java   |     2 +-
 .../messaging/event/TaskOutputChangeEvent.java  |     2 +-
 .../messaging/event/TaskStatusChangeEvent.java  |     2 +-
 .../event/TaskStatusChangeRequestEvent.java     |     2 +-
 .../airavata/model/process/ProcessModel.java    |     2 +-
 .../ComputationalResourceSchedulingModel.java   |     2 +-
 .../airavata/model/security/AuthzToken.java     |     2 +-
 .../airavata/model/status/ExperimentStatus.java |     2 +-
 .../apache/airavata/model/status/JobStatus.java |     2 +-
 .../airavata/model/status/ProcessStatus.java    |     2 +-
 .../airavata/model/status/TaskStatus.java       |     2 +-
 .../model/task/DataStagingTaskModel.java        |     2 +-
 .../model/task/EnvironmentSetupTaskModel.java   |     2 +-
 .../model/task/JobSubmissionTaskModel.java      |     2 +-
 .../airavata/model/task/MonitorTaskModel.java   |     2 +-
 .../apache/airavata/model/task/TaskModel.java   |     2 +-
 .../airavata/model/workspace/Gateway.java       |     2 +-
 .../apache/airavata/model/workspace/Group.java  |     2 +-
 .../airavata/model/workspace/Project.java       |     2 +-
 .../apache/airavata/model/workspace/User.java   |     2 +-
 .../main/resources/airavata-server.properties   |    14 +-
 .../test/resources/airavata-server.properties   |    16 +-
 .../apache/airavata/gfac/core/GFacUtils.java    |    19 +-
 .../server/OrchestratorServerHandler.java       |    12 +-
 .../core/data/catalog/impl/DataCatalogImpl.java |   355 -
 .../core/data/catalog/model/Configuration.java  |    55 -
 .../core/data/catalog/model/DataProduct.java    |   187 -
 .../data/catalog/model/DataProductMetaData.java |    77 -
 .../catalog/model/DataProductMetaData_PK.java   |    59 -
 .../data/catalog/model/DataReplicaLocation.java |   169 -
 .../data/catalog/model/DataReplicaMetaData.java |    77 -
 .../catalog/model/DataReplicaMetaData_PK.java   |    59 -
 .../catalog/utils/DataCatalogConstants.java     |    49 -
 .../data/catalog/utils/DataCatalogJPAUtils.java |    82 -
 .../utils/DataCatalogQueryGenerator.java        |   111 -
 .../utils/ThriftDataModelConversion.java        |   215 -
 .../catalog/impl/ExperimentRegistry.java        |     9 +-
 .../catalog/impl/RegistryFactory.java           |    10 +-
 .../catalog/model/ExperimentInput.java          |    10 +
 .../catalog/model/ExperimentOutput.java         |    11 +
 .../experiment/catalog/model/ProcessInput.java  |    11 +
 .../experiment/catalog/model/ProcessOutput.java |    11 +
 .../resources/ExperimentInputResource.java      |    10 +
 .../resources/ExperimentOutputResource.java     |    10 +
 .../catalog/resources/ProcessInputResource.java |    10 +
 .../resources/ProcessOutputResource.java        |    10 +
 .../utils/ThriftDataModelConversion.java        |     4 +
 .../registry/core/impl/RegistryImpl.java        |     6 +-
 .../catalog/impl/ReplicaCatalogImpl.java        |   366 +
 .../replica/catalog/model/Configuration.java    |    55 +
 .../core/replica/catalog/model/DataProduct.java |   177 +
 .../catalog/model/DataProductMetaData.java      |    77 +
 .../catalog/model/DataProductMetaData_PK.java   |    59 +
 .../catalog/model/DataReplicaLocation.java      |   169 +
 .../catalog/model/DataReplicaMetaData.java      |    77 +
 .../catalog/model/DataReplicaMetaData_PK.java   |    59 +
 .../catalog/utils/ReplicaCatalogConstants.java  |    49 +
 .../catalog/utils/ReplicaCatalogJPAUtils.java   |    82 +
 .../utils/ReplicaCatalogQueryGenerator.java     |   111 +
 .../utils/ThriftDataModelConversion.java        |   203 +
 .../src/main/resources/META-INF/persistence.xml |    12 +-
 .../src/main/resources/datacatalog-derby.sql    |    78 -
 .../src/main/resources/datacatalog-mysql.sql    |    77 -
 .../src/main/resources/expcatalog-derby.sql     |     4 +
 .../src/main/resources/expcatalog-mysql.sql     |     4 +
 .../src/main/resources/replicacatalog-derby.sql |    77 +
 .../src/main/resources/replicacatalog-mysql.sql |    76 +
 .../airavata/data/catalog/DataCatalogTest.java  |   100 -
 .../airavata/data/catalog/util/Initialize.java  |   315 -
 .../replica/catalog/ReplicaCatalogTest.java     |    99 +
 .../replica/catalog/util/Initialize.java        |   315 +
 .../airavata/registry/cpi/DataCatalog.java      |    50 -
 .../registry/cpi/DataCatalogException.java      |    35 -
 .../apache/airavata/registry/cpi/Registry.java  |     2 +-
 .../airavata/registry/cpi/ReplicaCatalog.java   |    54 +
 .../registry/cpi/ReplicaCatalogException.java   |    35 +
 .../airavata-apis/airavata_api.thrift           |    21 +-
 .../data-models/airavata_data_models.thrift     |     2 +-
 .../application_io_models.thrift                |     6 +-
 .../data_catalog_models.thrift                  |    73 -
 .../replica_catalog_models.thrift               |    70 +
 179 files changed, 39224 insertions(+), 32067 deletions(-)
----------------------------------------------------------------------