You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by sh...@apache.org on 2016/03/25 22:23:07 UTC
[44/55] [abbrv] airavata git commit: setting replica URIs for
experiment outputs
http://git-wip-us.apache.org/repos/asf/airavata/blob/8e4212ea/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/workspace/Group.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/workspace/Group.java b/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/workspace/Group.java
index 7bc9899..27d3cff 100644
--- a/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/workspace/Group.java
+++ b/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/workspace/Group.java
@@ -51,7 +51,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2016-03-24")
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2016-03-25")
public class Group implements org.apache.thrift.TBase<Group, Group._Fields>, java.io.Serializable, Cloneable, Comparable<Group> {
private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("Group");
http://git-wip-us.apache.org/repos/asf/airavata/blob/8e4212ea/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/workspace/Project.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/workspace/Project.java b/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/workspace/Project.java
index 8dd871b..4c6a844 100644
--- a/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/workspace/Project.java
+++ b/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/workspace/Project.java
@@ -51,7 +51,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2016-03-24")
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2016-03-25")
public class Project implements org.apache.thrift.TBase<Project, Project._Fields>, java.io.Serializable, Cloneable, Comparable<Project> {
private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("Project");
http://git-wip-us.apache.org/repos/asf/airavata/blob/8e4212ea/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/workspace/User.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/workspace/User.java b/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/workspace/User.java
index d21c7ec..641abc3 100644
--- a/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/workspace/User.java
+++ b/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/workspace/User.java
@@ -51,7 +51,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
-@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2016-03-24")
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)", date = "2016-03-25")
public class User implements org.apache.thrift.TBase<User, User._Fields>, java.io.Serializable, Cloneable, Comparable<User> {
private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("User");
http://git-wip-us.apache.org/repos/asf/airavata/blob/8e4212ea/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
index aa44fc6..65085db 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
@@ -35,6 +35,7 @@ import org.apache.airavata.model.application.io.DataType;
import org.apache.airavata.model.application.io.InputDataObjectType;
import org.apache.airavata.model.application.io.OutputDataObjectType;
import org.apache.airavata.model.commons.ErrorModel;
+import org.apache.airavata.model.data.replica.*;
import org.apache.airavata.model.experiment.ExperimentModel;
import org.apache.airavata.model.job.JobModel;
import org.apache.airavata.model.messaging.event.*;
@@ -862,7 +863,23 @@ public class GFacUtils {
if (experimentOutputs != null && !experimentOutputs.isEmpty()){
for (OutputDataObjectType expOutput : experimentOutputs){
if (expOutput.getName().equals(outputName)){
- expOutput.setValue(outputVal);
+ DataProductModel dataProductModel = new DataProductModel();
+ //FIXME We need to set the username here
+ dataProductModel.setGatewayId(processContext.getGatewayId());
+ dataProductModel.setProductName(outputName);
+ dataProductModel.setDataProductType(DataProductType.FILE);
+
+ DataReplicaLocationModel replicaLocationModel = new DataReplicaLocationModel();
+ replicaLocationModel.setStorageResourceId(processContext.getStorageResource().getStorageResourceId());
+ replicaLocationModel.setReplicaName(outputName + " gateway data store copy");
+ replicaLocationModel.setReplicaLocationCategory(ReplicaLocationCategory.GATEWAY_DATA_STORE);
+ replicaLocationModel.setReplicaPersistentType(ReplicaPersistentType.TRANSIENT);
+ replicaLocationModel.setFilePath(outputVal);
+ dataProductModel.addToReplicaLocations(replicaLocationModel);
+
+ ReplicaCatalog replicaCatalog = RegistryFactory.getReplicaCatalog();
+ String productUri = replicaCatalog.registerDataProduct(dataProductModel);
+ expOutput.setValue(productUri);
}
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/8e4212ea/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/replica/catalog/impl/ReplicaCatalogImpl.java
----------------------------------------------------------------------
diff --git a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/replica/catalog/impl/ReplicaCatalogImpl.java b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/replica/catalog/impl/ReplicaCatalogImpl.java
index b946710..c35a3bd 100644
--- a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/replica/catalog/impl/ReplicaCatalogImpl.java
+++ b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/replica/catalog/impl/ReplicaCatalogImpl.java
@@ -34,8 +34,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.persistence.EntityManager;
-import java.net.URI;
-import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -47,58 +45,17 @@ public class ReplicaCatalogImpl implements ReplicaCatalog {
@Override
public String registerDataProduct(DataProductModel productModel) throws ReplicaCatalogException {
- if(productModel.getOwnerName() == null || productModel.getGatewayId() == null || productModel
- .getLogicalPath() == null || !productModel.getLogicalPath().startsWith("/")){
- throw new ReplicaCatalogException("owner name, gateway id and logical path should be non empty and logical path" +
- " should start with /");
+ if(productModel.getOwnerName() == null || productModel.getGatewayId() == null){
+ throw new ReplicaCatalogException("owner name and gateway id should be non empty");
}
- if(productModel.getDataProductType().equals(DataProductType.FILE) && !productModel.getLogicalPath().endsWith(productModel.getProductName())){
- if(!productModel.getLogicalPath().endsWith("/"))
- productModel.setLogicalPath(productModel.getLogicalPath()+"/");
- productModel.setLogicalPath(productModel.getLogicalPath()+productModel.getProductName());
- }
- //Creating parent logical dir if not exist too
- String parentUri = ReplicaCatalog.schema + "://" + productModel.getOwnerName() + "@" + productModel.getGatewayId() + ":/" ;
- DataProductModel tempDp;
- final long currentTime = System.currentTimeMillis();
- if(!isExists(parentUri)){
- tempDp = new DataProductModel();
- tempDp.setProductUri(parentUri);
- tempDp.setLogicalPath("/");
- tempDp.setOwnerName(productModel.getOwnerName());
- tempDp.setGatewayId(productModel.getGatewayId());
- tempDp.setProductName("/");
- tempDp.setCreationTime(currentTime);
- tempDp.setLastModifiedTime(currentTime);
- tempDp.setDataProductType(DataProductType.DIR);
- createDataProduct(tempDp);
- }
- String[] bits = productModel.getLogicalPath().split("/");
- for(int i=0; i<bits.length-1;i++){
- String dir = bits[i];
- if(!isExists(parentUri + dir)){
- tempDp = new DataProductModel();
- try {
- if(!parentUri.endsWith("/")) dir = "/" + dir;
- tempDp.setLogicalPath((new URI(parentUri + dir)).getPath());
- } catch (URISyntaxException e) {
- throw new ReplicaCatalogException(e);
- }
- tempDp.setProductUri(parentUri + dir);
- tempDp.setOwnerName(productModel.getOwnerName());
- tempDp.setGatewayId(productModel.getGatewayId());
- tempDp.setProductName(dir.substring(1));
- tempDp.setCreationTime(currentTime);
- tempDp.setLastModifiedTime(currentTime);
- tempDp.setDataProductType(DataProductType.DIR);
- tempDp.setParentProductUri(parentUri);
- parentUri = createDataProduct(tempDp);
- }
+
+ if(productModel.getParentProductUri() != null && (!isExists(productModel.getParentProductUri())
+ || !getDataProduct(productModel.getParentProductUri()).getDataProductType().equals(DataProductType.COLLECTION))){
+ throw new ReplicaCatalogException("Parent Product does not exists or parent type is not Collection");
}
- productModel.setParentProductUri(parentUri);
- String productUri = ReplicaCatalog.schema + "://" + productModel.getOwnerName() + "@" + productModel.getGatewayId()
- + ":" + productModel.getLogicalPath();
+ final long currentTime = System.currentTimeMillis();
+ String productUri = ReplicaCatalog.schema + "://" + UUID.randomUUID().toString();
productModel.setProductUri(productUri);
productModel.setCreationTime(currentTime);
productModel.setLastModifiedTime(currentTime);
http://git-wip-us.apache.org/repos/asf/airavata/blob/8e4212ea/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/replica/catalog/model/DataProduct.java
----------------------------------------------------------------------
diff --git a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/replica/catalog/model/DataProduct.java b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/replica/catalog/model/DataProduct.java
index 8052c2b..f91d4bf 100644
--- a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/replica/catalog/model/DataProduct.java
+++ b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/replica/catalog/model/DataProduct.java
@@ -34,7 +34,6 @@ public class DataProduct {
private String productUri;
private String gatewayId;
private String productName;
- private String logicalPath;
private String productDescription;
private String dataProductType;
private String ownerName;
@@ -85,15 +84,6 @@ public class DataProduct {
this.productDescription = productDescription;
}
- @Column(name = "LOGICAL_PATH")
- public String getLogicalPath() {
- return logicalPath;
- }
-
- public void setLogicalPath(String logicalPath) {
- this.logicalPath = logicalPath;
- }
-
@Column(name = "OWNER_NAME")
public String getOwnerName() {
return ownerName;
http://git-wip-us.apache.org/repos/asf/airavata/blob/8e4212ea/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/replica/catalog/utils/ThriftDataModelConversion.java
----------------------------------------------------------------------
diff --git a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/replica/catalog/utils/ThriftDataModelConversion.java b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/replica/catalog/utils/ThriftDataModelConversion.java
index 741215a..aba37c9 100644
--- a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/replica/catalog/utils/ThriftDataModelConversion.java
+++ b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/replica/catalog/utils/ThriftDataModelConversion.java
@@ -43,7 +43,6 @@ public class ThriftDataModelConversion {
if (dataProduct != null) {
DataProductModel dataProductModel = new DataProductModel();
dataProductModel.setProductUri(dataProduct.getProductUri());
- dataProductModel.setLogicalPath(dataProduct.getLogicalPath());
dataProductModel.setGatewayId(dataProduct.getGatewayId());
dataProductModel.setParentProductUri(dataProduct.getParentProductUri());
dataProductModel.setProductName(dataProduct.getProductName());
@@ -80,7 +79,6 @@ public class ThriftDataModelConversion {
public static DataProduct getUpdatedDataProduct(DataProductModel dataProductModel, DataProduct dataProduct){
dataProduct.setProductUri(dataProductModel.getProductUri());
- dataProduct.setLogicalPath(dataProductModel.getLogicalPath());
dataProduct.setGatewayId(dataProductModel.getGatewayId());
dataProduct.setProductName(dataProductModel.getProductName());
dataProduct.setParentProductUri(dataProductModel.getParentProductUri());
http://git-wip-us.apache.org/repos/asf/airavata/blob/8e4212ea/modules/registry/registry-core/src/main/resources/replicacatalog-derby.sql
----------------------------------------------------------------------
diff --git a/modules/registry/registry-core/src/main/resources/replicacatalog-derby.sql b/modules/registry/registry-core/src/main/resources/replicacatalog-derby.sql
index 78dc165..f510f36 100644
--- a/modules/registry/registry-core/src/main/resources/replicacatalog-derby.sql
+++ b/modules/registry/registry-core/src/main/resources/replicacatalog-derby.sql
@@ -26,7 +26,6 @@ CREATE TABLE DATA_PRODUCT
PRODUCT_NAME VARCHAR (255),
PRODUCT_DESCRIPTION VARCHAR (1024),
PARENT_PRODUCT_URI VARCHAR (255),
- LOGICAL_PATH VARCHAR (255),
OWNER_NAME VARCHAR (255),
PRODUCT_SIZE INTEGER ,
CREATION_TIME TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
http://git-wip-us.apache.org/repos/asf/airavata/blob/8e4212ea/modules/registry/registry-core/src/main/resources/replicacatalog-mysql.sql
----------------------------------------------------------------------
diff --git a/modules/registry/registry-core/src/main/resources/replicacatalog-mysql.sql b/modules/registry/registry-core/src/main/resources/replicacatalog-mysql.sql
index d612194..bb9bbfc 100644
--- a/modules/registry/registry-core/src/main/resources/replicacatalog-mysql.sql
+++ b/modules/registry/registry-core/src/main/resources/replicacatalog-mysql.sql
@@ -24,7 +24,6 @@ CREATE TABLE DATA_PRODUCT
PRODUCT_URI VARCHAR (255),
GATEWAY_ID VARCHAR (255),
PRODUCT_NAME VARCHAR (255),
- LOGICAL_PATH VARCHAR (255),
PRODUCT_DESCRIPTION VARCHAR (255),
OWNER_NAME VARCHAR (255),
PARENT_PRODUCT_URI VARCHAR (255),
http://git-wip-us.apache.org/repos/asf/airavata/blob/8e4212ea/modules/registry/registry-core/src/test/java/org/apache/airavata/replica/catalog/ReplicaCatalogTest.java
----------------------------------------------------------------------
diff --git a/modules/registry/registry-core/src/test/java/org/apache/airavata/replica/catalog/ReplicaCatalogTest.java b/modules/registry/registry-core/src/test/java/org/apache/airavata/replica/catalog/ReplicaCatalogTest.java
index 1bfa014..a2e2abf 100644
--- a/modules/registry/registry-core/src/test/java/org/apache/airavata/replica/catalog/ReplicaCatalogTest.java
+++ b/modules/registry/registry-core/src/test/java/org/apache/airavata/replica/catalog/ReplicaCatalogTest.java
@@ -52,7 +52,6 @@ public class ReplicaCatalogTest {
dataProductModel.setProductName("test-file.txt");
dataProductModel.setOwnerName("scnakandala");
dataProductModel.setGatewayId("default");
- dataProductModel.setLogicalPath("/test/test/test");
dataProductModel.setDataProductType(DataProductType.FILE);
HashMap<String, String> resMetadata = new HashMap<>();
resMetadata.put("name", "name");
http://git-wip-us.apache.org/repos/asf/airavata/blob/8e4212ea/modules/registry/registry-cpi/src/main/java/org/apache/airavata/registry/cpi/ReplicaCatalogException.java
----------------------------------------------------------------------
diff --git a/modules/registry/registry-cpi/src/main/java/org/apache/airavata/registry/cpi/ReplicaCatalogException.java b/modules/registry/registry-cpi/src/main/java/org/apache/airavata/registry/cpi/ReplicaCatalogException.java
index f0eb5cd..3b49aed 100644
--- a/modules/registry/registry-cpi/src/main/java/org/apache/airavata/registry/cpi/ReplicaCatalogException.java
+++ b/modules/registry/registry-cpi/src/main/java/org/apache/airavata/registry/cpi/ReplicaCatalogException.java
@@ -19,7 +19,7 @@
package org.apache.airavata.registry.cpi;
-public class ReplicaCatalogException extends Exception{
+public class ReplicaCatalogException extends RegistryException{
public ReplicaCatalogException(Throwable e) {
super(e);
http://git-wip-us.apache.org/repos/asf/airavata/blob/8e4212ea/thrift-interface-descriptions/data-models/replica-catalog-models/replica_catalog_models.thrift
----------------------------------------------------------------------
diff --git a/thrift-interface-descriptions/data-models/replica-catalog-models/replica_catalog_models.thrift b/thrift-interface-descriptions/data-models/replica-catalog-models/replica_catalog_models.thrift
index cd41eda..b7fdff7 100644
--- a/thrift-interface-descriptions/data-models/replica-catalog-models/replica_catalog_models.thrift
+++ b/thrift-interface-descriptions/data-models/replica-catalog-models/replica_catalog_models.thrift
@@ -35,7 +35,6 @@ enum ReplicaPersistentType {
}
enum DataProductType {
- DIR,
FILE,
COLLECTION,
}
@@ -44,16 +43,15 @@ struct DataProductModel {
1: optional string productUri,
2: optional string gatewayId,
3: optional string parentProductUri,
- 4: optional string logicalPath,
- 5: optional string productName,
- 6: optional string productDescription,
- 7: optional string ownerName,
- 8: optional DataProductType dataProductType,
- 9: optional i32 productSize,
- 10: optional i64 creationTime,
- 11: optional i64 lastModifiedTime,
- 12: optional map<string, string> productMetadata,
- 13: optional list<DataReplicaLocationModel> replicaLocations,
+ 4: optional string productName,
+ 5: optional string productDescription,
+ 6: optional string ownerName,
+ 7: optional DataProductType dataProductType,
+ 8: optional i32 productSize,
+ 9: optional i64 creationTime,
+ 10: optional i64 lastModifiedTime,
+ 11: optional map<string, string> productMetadata,
+ 12: optional list<DataReplicaLocationModel> replicaLocations,
}
struct DataReplicaLocationModel {