You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by sc...@apache.org on 2015/12/02 23:55:37 UTC
[40/44] airavata git commit: adding the initial version of the data
catalog
http://git-wip-us.apache.org/repos/asf/airavata/blob/1a22de16/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/impl/DataCatalogImpl.java
----------------------------------------------------------------------
diff --git a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/impl/DataCatalogImpl.java b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/impl/DataCatalogImpl.java
new file mode 100644
index 0000000..98dc8c3
--- /dev/null
+++ b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/impl/DataCatalogImpl.java
@@ -0,0 +1,149 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.registry.core.data.catalog.impl;
+
+import org.apache.airavata.model.data.resource.DataReplicaLocationModel;
+import org.apache.airavata.model.data.resource.DataResourceModel;
+import org.apache.airavata.registry.core.data.catalog.model.DataResource;
+import org.apache.airavata.registry.core.data.catalog.utils.DataCatalogJPAUtils;
+import org.apache.airavata.registry.core.data.catalog.utils.ThriftDataModelConversion;
+import org.apache.airavata.registry.cpi.DataCatalog;
+import org.apache.airavata.registry.cpi.DataCatalogException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.persistence.EntityManager;
+import java.util.UUID;
+
+public class DataCatalogImpl implements DataCatalog {
+
+ private final static Logger logger = LoggerFactory.getLogger(DataCatalogImpl.class);
+
+ @Override
+ public String publishResource(DataResourceModel resourceModel) throws DataCatalogException {
+ String resourceId = UUID.randomUUID().toString();
+ resourceModel.setResourceId(resourceId);
+ for(DataReplicaLocationModel replicaLocationModel : resourceModel.getReplicaLocations()){
+ replicaLocationModel.setReplicaId(UUID.randomUUID().toString());
+ replicaLocationModel.setResourceId(resourceId);
+ }
+ DataResource dataResource = ThriftDataModelConversion.getDataResource(resourceModel);
+ EntityManager em = null;
+ try {
+ em = DataCatalogJPAUtils.getEntityManager();
+ em.getTransaction().begin();
+ em.persist(dataResource);
+ em.getTransaction().commit();
+ em.close();
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ throw new DataCatalogException(e);
+ } finally {
+ if (em != null && em.isOpen()) {
+ if (em.getTransaction().isActive()) {
+ em.getTransaction().rollback();
+ }
+ em.close();
+ }
+ }
+ return resourceId;
+ }
+
+ @Override
+ public boolean removeResource(String resourceId) throws DataCatalogException {
+ EntityManager em = null;
+ try {
+ em = DataCatalogJPAUtils.getEntityManager();
+ DataResource dataResource = em.find(DataResource.class, resourceId);
+ if(dataResource == null)
+ return false;
+ em.getTransaction().begin();
+ em.remove(dataResource);
+ em.getTransaction().commit();
+ em.close();
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ throw new DataCatalogException(e);
+ } finally {
+ if (em != null && em.isOpen()) {
+ if (em.getTransaction().isActive()) {
+ em.getTransaction().rollback();
+ }
+ em.close();
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public boolean updateResource(DataResourceModel resourceModel) throws DataCatalogException {
+ EntityManager em = null;
+ try {
+ em = DataCatalogJPAUtils.getEntityManager();
+ DataResource dataResource = em.find(DataResource.class, resourceModel.getResourceId());
+ if(dataResource == null)
+ return false;
+ //FIXME - Every time we delete and create
+ em.getTransaction().begin();
+ em.remove(dataResource);
+ em.flush();
+ resourceModel.getReplicaLocations().stream().forEach(rl->{
+ rl.setReplicaId(UUID.randomUUID().toString());
+ rl.setResourceId(resourceModel.getResourceId());
+ });
+ em.persist(ThriftDataModelConversion.getDataResource(resourceModel));
+ em.getTransaction().commit();
+ em.close();
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ throw new DataCatalogException(e);
+ } finally {
+ if (em != null && em.isOpen()) {
+ if (em.getTransaction().isActive()) {
+ em.getTransaction().rollback();
+ }
+ em.close();
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public DataResourceModel getResource(String resourceId) throws DataCatalogException {
+ EntityManager em = null;
+ try {
+ em = DataCatalogJPAUtils.getEntityManager();
+ DataResource dataResource = em.find(DataResource.class, resourceId);
+ return ThriftDataModelConversion.getDataResourceModel(dataResource);
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ throw new DataCatalogException(e);
+ } finally {
+ if (em != null && em.isOpen()) {
+ if (em.getTransaction().isActive()) {
+ em.getTransaction().rollback();
+ }
+ em.close();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/1a22de16/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/model/Configuration.java
----------------------------------------------------------------------
diff --git a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/model/Configuration.java b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/model/Configuration.java
new file mode 100644
index 0000000..e4ad0ec
--- /dev/null
+++ b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/model/Configuration.java
@@ -0,0 +1,55 @@
+/*
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*/
+package org.apache.airavata.registry.core.data.catalog.model;
+
+import org.apache.airavata.registry.core.app.catalog.model.Configuration_PK;
+
+import javax.persistence.*;
+import java.io.Serializable;
+
+@Entity
+@Table(name ="CONFIGURATION")
+@IdClass(Configuration_PK.class)
+public class Configuration implements Serializable {
+ @Id
+ @Column(name = "CONFIG_KEY")
+ private String config_key;
+
+ @Id
+ @Column(name = "CONFIG_VAL")
+ private String config_val;
+
+ public String getConfig_key() {
+ return config_key;
+ }
+
+ public String getConfig_val() {
+ return config_val;
+ }
+
+ public void setConfig_key(String config_key) {
+ this.config_key = config_key;
+ }
+
+ public void setConfig_val(String config_val) {
+ this.config_val = config_val;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/1a22de16/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/model/DataReplicaLocation.java
----------------------------------------------------------------------
diff --git a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/model/DataReplicaLocation.java b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/model/DataReplicaLocation.java
new file mode 100644
index 0000000..40b23a7
--- /dev/null
+++ b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/model/DataReplicaLocation.java
@@ -0,0 +1,111 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.registry.core.data.catalog.model;
+
+import com.mysql.fabric.xmlrpc.base.Data;
+import org.apache.airavata.registry.core.experiment.catalog.model.Experiment;
+import org.apache.airavata.registry.core.experiment.catalog.model.Gateway;
+import org.apache.airavata.registry.core.experiment.catalog.model.ProjectUser;
+import org.apache.airavata.registry.core.experiment.catalog.model.Users;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.persistence.*;
+import java.sql.Timestamp;
+import java.util.Collection;
+
+@Entity
+@Table(name = "DATA_REPLICA_LOCATION")
+public class DataReplicaLocation {
+ private final static Logger logger = LoggerFactory.getLogger(DataReplicaLocation.class);
+ private String resourceId;
+ private String replicaName;
+ private String replicaDescription;
+ private String dataLocations;
+ private Timestamp creationTime;
+ private Timestamp lastModifiedTime;
+
+ private DataResource dataResource;
+
+ @Column(name = "RESOURCE_ID")
+ public String getResourceId() {
+ return resourceId;
+ }
+
+ public void setResourceId(String resourceId) {
+ this.resourceId = resourceId;
+ }
+
+ @Column(name = "DATA_LOCATIONS")
+ public String getDataLocations() {
+ return dataLocations;
+ }
+
+ public void setDataLocations(String dataLocations) {
+ this.dataLocations = dataLocations;
+ }
+
+ @Column(name = "REPLICA_NAME")
+ public String getReplicaName() {
+ return replicaName;
+ }
+
+ public void setReplicaName(String replicaName) {
+ this.replicaName = replicaName;
+ }
+
+ @Column(name = "REPLICA_DESCRIPTION")
+ public String getReplicaDescription() {
+ return replicaDescription;
+ }
+
+ public void setReplicaDescription(String replicaDescription) {
+ this.replicaDescription = replicaDescription;
+ }
+
+ @Column(name = "CREATION_TIME")
+ public Timestamp getCreationTime() {
+ return creationTime;
+ }
+
+ public void setCreationTime(Timestamp creationTime) {
+ this.creationTime = creationTime;
+ }
+
+ @Column(name = "LAST_MODIFIED_TIME")
+ public Timestamp getLastModifiedTime() {
+ return lastModifiedTime;
+ }
+
+ public void setLastModifiedTime(Timestamp lastModifiedTime) {
+ this.lastModifiedTime = lastModifiedTime;
+ }
+
+ @ManyToOne
+ @JoinColumn(name = "RESOURCE_ID", referencedColumnName = "RESOURCE_ID")
+ public DataResource getDataResource() {
+ return dataResource;
+ }
+
+ public void setDataResource(DataResource dataResource) {
+ this.dataResource = dataResource;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/1a22de16/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/model/DataResource.java
----------------------------------------------------------------------
diff --git a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/model/DataResource.java b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/model/DataResource.java
new file mode 100644
index 0000000..dd1914f
--- /dev/null
+++ b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/model/DataResource.java
@@ -0,0 +1,108 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.registry.core.data.catalog.model;
+
+import org.apache.airavata.registry.core.experiment.catalog.model.GatewayWorker;
+import org.apache.airavata.registry.core.experiment.catalog.model.Project;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.persistence.*;
+import java.sql.Timestamp;
+import java.util.Collection;
+
+@Entity
+@Table(name = "DATA_RESOURCE")
+public class DataResource {
+ private final static Logger logger = LoggerFactory.getLogger(DataResource.class);
+ private String resourceId;
+ private String resourceName;
+ private String resourceDescription;
+ private int resourceSize;
+ private Timestamp creationTime;
+ private Timestamp lastModifiedTime;
+
+ private Collection<DataReplicaLocation> dataReplicaLocations;
+
+ @Id
+ @Column(name = "RESOURCE_ID")
+ public String getResourceId() {
+ return resourceId;
+ }
+
+ public void setResourceId(String resourceId) {
+ this.resourceId = resourceId;
+ }
+
+ @Column(name = "RESOURCE_NAME")
+ public String getResourceName() {
+ return resourceName;
+ }
+
+ public void setResourceName(String resourceName) {
+ this.resourceName = resourceName;
+ }
+
+ @Column(name = "RESOURCE_DESCRIPTION")
+ public String getResourceDescription() {
+ return resourceDescription;
+ }
+
+ public void setResourceDescription(String resourceDescription) {
+ this.resourceDescription = resourceDescription;
+ }
+
+ @Column(name = "RESOURCE_SIZE")
+ public int getResourceSize() {
+ return resourceSize;
+ }
+
+ public void setResourceSize(int resourceSize) {
+ this.resourceSize = resourceSize;
+ }
+
+ @Column(name = "CREATION_TIME")
+ public Timestamp getCreationTime() {
+ return creationTime;
+ }
+
+ public void setCreationTime(Timestamp creationTime) {
+ this.creationTime = creationTime;
+ }
+
+ @Column(name = "LAST_MODIFIED_TIME")
+ public Timestamp getLastModifiedTime() {
+ return lastModifiedTime;
+ }
+
+ public void setLastModifiedTime(Timestamp lastModifiedTime) {
+ this.lastModifiedTime = lastModifiedTime;
+ }
+
+ @OneToMany(mappedBy = "dataResource", cascade = {CascadeType.PERSIST, CascadeType.REMOVE})
+ public Collection<DataReplicaLocation> getDataReplicaLocations() {
+ return dataReplicaLocations;
+ }
+
+ public void setDataReplicaLocations(Collection<DataReplicaLocation> dataReplicaLocations) {
+ this.dataReplicaLocations = dataReplicaLocations;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/1a22de16/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/utils/DataCatalogConstants.java
----------------------------------------------------------------------
diff --git a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/utils/DataCatalogConstants.java b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/utils/DataCatalogConstants.java
new file mode 100644
index 0000000..c9e18f6
--- /dev/null
+++ b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/utils/DataCatalogConstants.java
@@ -0,0 +1,49 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.airavata.registry.core.data.catalog.utils;
+
+public class DataCatalogConstants {
+ // table names
+ public static final String DATA_RESOURCE = "DataResource";
+ public static final String DATA_REPLICA_LOCATION = "DataReplicaLocation";
+ public static final String CONFIGURATION = "Configuration";
+
+ // DataResource Table
+ public final class DataResourceConstants {
+ public static final String RESOURCE_ID = "resourceId";
+ public static final String RESOURCE_NAME = "resourceName";
+ public static final String RESOURCE_DESCRIPTION = "resourceDescription";
+ public static final String RESOURCE_SIZE = "resourceSize";
+ public static final String CREATION_TIME = "creationTime";
+ public static final String LAST_MODIFIED_TIME = "lastModifiedTime";
+ }
+
+ // Users table
+ public final class DataReplicaLocationConstants {
+ public static final String REPLICA_ID = "replicaId";
+ public static final String RESOURCE_ID = "resourceId";
+ public static final String DATA_LOCATIONS = "dataLocations";
+ public static final String REPLICA_NAME = "replicaName";
+ public static final String REPLICA_DESCRIPTION = "replicaDescription";
+ public static final String CREATION_TIME = "creationTime";
+ public static final String LAST_MODIFIED_TIME = "lastModifiedTime";
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/1a22de16/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/utils/DataCatalogJPAUtils.java
----------------------------------------------------------------------
diff --git a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/utils/DataCatalogJPAUtils.java b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/utils/DataCatalogJPAUtils.java
new file mode 100644
index 0000000..b8257cb
--- /dev/null
+++ b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/utils/DataCatalogJPAUtils.java
@@ -0,0 +1,82 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.registry.core.data.catalog.utils;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.persistence.*;
+import java.util.HashMap;
+import java.util.Map;
+
+public class DataCatalogJPAUtils {
+ private final static Logger logger = LoggerFactory.getLogger(DataCatalogJPAUtils.class);
+
+ private static final String PERSISTENCE_UNIT_NAME = "datacatalog_data";
+ private static final String DATACATALOG_JDBC_DRIVER = "datacatalog.jdbc.driver";
+ private static final String DATACATALOG_JDBC_URL = "datacatalog.jdbc.url";
+ private static final String DATACATALOG_JDBC_USER = "datacatalog.jdbc.user";
+ private static final String DATACATALOG_JDBC_PWD = "datacatalog.jdbc.password";
+ private static final String DATACATALOG_VALIDATION_QUERY = "datacatalog.validationQuery";
+
+ @PersistenceUnit(unitName="datacatalog_data")
+ protected static EntityManagerFactory factory;
+
+ @PersistenceContext(unitName="datacatalog_data")
+ private static EntityManager dataCatEntityManager;
+
+ public static EntityManager getEntityManager() throws ApplicationSettingsException {
+ if (factory == null) {
+ String connectionProperties = "DriverClassName=" + readServerProperties(DATACATALOG_JDBC_DRIVER) + "," +
+ "Url=" + readServerProperties(DATACATALOG_JDBC_URL) + "?autoReconnect=true," +
+ "Username=" + readServerProperties(DATACATALOG_JDBC_USER) + "," +
+ "Password=" + readServerProperties(DATACATALOG_JDBC_PWD) +
+ ",validationQuery=" + readServerProperties(DATACATALOG_VALIDATION_QUERY);
+ System.out.println(connectionProperties);
+ Map<String, String> properties = new HashMap<String, String>();
+ properties.put("openjpa.ConnectionDriverName", "org.apache.commons.dbcp.BasicDataSource");
+ properties.put("openjpa.ConnectionProperties", connectionProperties);
+ properties.put("openjpa.DynamicEnhancementAgent", "true");
+ properties.put("openjpa.RuntimeUnenhancedClasses", "unsupported");
+ properties.put("openjpa.RemoteCommitProvider","sjvm");
+ properties.put("openjpa.Log","DefaultLevel=INFO, Runtime=INFO, Tool=INFO, SQL=INFO");
+ properties.put("openjpa.jdbc.SynchronizeMappings", "buildSchema(ForeignKeys=true)");
+ properties.put("openjpa.jdbc.QuerySQLCache", "false");
+ properties.put("openjpa.ConnectionFactoryProperties", "PrettyPrint=true, PrettyPrintLineLength=72," +
+ " PrintParameters=true, MaxActive=10, MaxIdle=5, MinIdle=2, MaxWait=31536000, autoReconnect=true");
+ factory = Persistence.createEntityManagerFactory(PERSISTENCE_UNIT_NAME, properties);
+ }
+ dataCatEntityManager = factory.createEntityManager();
+ return dataCatEntityManager;
+ }
+
+ private static String readServerProperties (String propertyName) throws ApplicationSettingsException {
+ try {
+ return ServerSettings.getSetting(propertyName);
+ } catch (ApplicationSettingsException e) {
+ logger.error("Unable to read airavata-server.properties...", e);
+ throw new ApplicationSettingsException("Unable to read airavata-server.properties...");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/1a22de16/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/utils/DataCatalogQueryGenerator.java
----------------------------------------------------------------------
diff --git a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/utils/DataCatalogQueryGenerator.java b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/utils/DataCatalogQueryGenerator.java
new file mode 100644
index 0000000..4ddf8d3
--- /dev/null
+++ b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/utils/DataCatalogQueryGenerator.java
@@ -0,0 +1,111 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.registry.core.data.catalog.utils;
+
+import org.apache.airavata.registry.cpi.ResultOrderType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.persistence.EntityManager;
+import javax.persistence.Query;
+import java.util.HashMap;
+import java.util.Map;
+
+public class DataCatalogQueryGenerator {
+
+ private final static Logger logger = LoggerFactory.getLogger(DataCatalogQueryGenerator.class);
+ private String tableName;
+ private Map<String,Object> matches=new HashMap<String, Object>();
+ private static final String SELECT_OBJ="p";
+ private static final String DELETE_OBJ="p";
+ private static final String TABLE_OBJ="p";
+
+ public DataCatalogQueryGenerator(String tableName, Object[]... params) {
+ setTableName(tableName);
+ for (Object[] param : params) {
+ addMatch(param[0].toString(), param[1]);
+ }
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public void addMatch(String colName, Object matchValue){
+ matches.put(colName, matchValue);
+ }
+
+ public void setParameter(String colName, Object matchValue){
+ addMatch(colName, matchValue);
+ }
+
+ public Query selectQuery(EntityManager entityManager){
+ String queryString="SELECT "+ SELECT_OBJ + " FROM " +getTableName()+" "+TABLE_OBJ;
+ return generateQueryWithParameters(entityManager, queryString);
+ }
+
+ public Query selectQuery(EntityManager entityManager, String orderByColumn,
+ ResultOrderType resultOrderType){
+ String order = (resultOrderType == ResultOrderType.ASC) ? "ASC" : "DESC";
+ String orderByClause = " ORDER BY " + SELECT_OBJ + "." + orderByColumn + " " + order;
+ String queryString="SELECT "+ SELECT_OBJ + " FROM " +getTableName()+" "+TABLE_OBJ;
+ return generateQueryWithParameters(entityManager, queryString, orderByClause);
+ }
+
+ public Query deleteQuery(EntityManager entityManager){
+ String queryString="Delete FROM "+getTableName()+" "+TABLE_OBJ;
+ return generateQueryWithParameters(entityManager, queryString);
+ }
+
+ private Query generateQueryWithParameters(EntityManager entityManager, String queryString) {
+ return generateQueryWithParameters(entityManager, queryString, "");
+ }
+
+ private Query generateQueryWithParameters(EntityManager entityManager,
+ String queryString, String orderByClause) {
+ Map<String,Object> queryParameters=new HashMap<String, Object>();
+ if (matches.size()>0){
+ String matchString = "";
+ int paramCount=0;
+ for (String colName : matches.keySet()) {
+ String paramName="param"+paramCount;
+ queryParameters.put(paramName, matches.get(colName));
+ if (!matchString.equals("")){
+ matchString+=" AND ";
+ }
+ matchString+=TABLE_OBJ+"."+colName+" =:"+paramName;
+ paramCount++;
+ }
+ queryString+=" WHERE "+matchString;
+ }
+ queryString += orderByClause;
+ Query query = entityManager.createQuery(queryString);
+ for (String paramName : queryParameters.keySet()) {
+ query.setParameter(paramName, queryParameters.get(paramName));
+ }
+ return query;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/1a22de16/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/utils/ThriftDataModelConversion.java
----------------------------------------------------------------------
diff --git a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/utils/ThriftDataModelConversion.java b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/utils/ThriftDataModelConversion.java
new file mode 100644
index 0000000..2ec041e
--- /dev/null
+++ b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/data/catalog/utils/ThriftDataModelConversion.java
@@ -0,0 +1,105 @@
+/*
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*/
+
+package org.apache.airavata.registry.core.data.catalog.utils;
+
+import org.apache.airavata.model.data.resource.DataReplicaLocationModel;
+import org.apache.airavata.model.data.resource.DataResourceModel;
+import org.apache.airavata.registry.core.data.catalog.model.DataReplicaLocation;
+import org.apache.airavata.registry.core.data.catalog.model.DataResource;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Timestamp;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class ThriftDataModelConversion {
+
+ private final static Logger logger = LoggerFactory.getLogger(ThriftDataModelConversion.class);
+
+ public static DataResourceModel getDataResourceModel(DataResource dataResource){
+ if (dataResource != null) {
+ DataResourceModel dataResourceModel = new DataResourceModel();
+ dataResourceModel.setResourceId(dataResource.getResourceId());
+ dataResourceModel.setResourceName(dataResource.getResourceName());
+ dataResourceModel.setResourceDescription(dataResource.getResourceDescription());
+ dataResourceModel.setResourceSize(dataResource.getResourceSize());
+ dataResourceModel.setCreationTime(dataResource.getCreationTime().getTime());
+ dataResourceModel.setLastModifiedTime(dataResource.getLastModifiedTime().getTime());
+ dataResource.getDataReplicaLocations().stream().forEach(rl->dataResourceModel
+ .addToReplicaLocations(getDataReplicaLocationModel(rl)));
+ return dataResourceModel;
+ }
+ return null;
+ }
+
+ public static DataResource getDataResource(DataResourceModel dataResourceModel){
+ if(dataResourceModel != null){
+ DataResource dataResource = new DataResource();
+ dataResource.setResourceId(dataResourceModel.getResourceId());
+ dataResource.setResourceName(dataResourceModel.getResourceName());
+ dataResource.setResourceDescription(dataResourceModel.getResourceDescription());
+ dataResource.setResourceSize(dataResourceModel.getResourceSize());
+ dataResource.setCreationTime(new Timestamp(dataResourceModel.getCreationTime()));
+ dataResource.setLastModifiedTime(new Timestamp(dataResourceModel.getLastModifiedTime()));
+ List<DataReplicaLocation> replicaLocationList = dataResourceModel.getReplicaLocations()
+ .stream().map(ThriftDataModelConversion::getDataReplicaLocation).collect(Collectors.toList());
+ dataResource.setDataReplicaLocations(replicaLocationList);
+ return dataResource;
+ }
+ return null;
+ }
+
+ public static DataReplicaLocationModel getDataReplicaLocationModel(DataReplicaLocation replicaLocation){
+ if (replicaLocation != null) {
+ DataReplicaLocationModel replicaLocationModel = new DataReplicaLocationModel();
+ replicaLocationModel.setResourceId(replicaLocation.getResourceId());
+ replicaLocationModel.setReplicaName(replicaLocation.getReplicaName());
+ replicaLocationModel.setReplicaDescription(replicaLocation.getReplicaDescription());
+ replicaLocationModel.setCreationTime(replicaLocation.getCreationTime().getTime());
+ replicaLocationModel.setLastModifiedTime(replicaLocation.getLastModifiedTime().getTime());
+ if(replicaLocation.getDataLocations()!=null && !replicaLocation.getDataLocations().isEmpty()) {
+ String[] dataLocations = replicaLocation.getDataLocations().split(",");
+ for(String dataLocation : dataLocations){
+ replicaLocationModel.addToDataLocations(dataLocation);
+ }
+ }
+ return replicaLocationModel;
+ }
+ return null;
+ }
+
+ public static DataReplicaLocation getDataReplicaLocation(DataReplicaLocationModel dataReplicaLocationModel){
+ if(dataReplicaLocationModel != null){
+ DataReplicaLocation dataReplicaLocation = new DataReplicaLocation();
+ dataReplicaLocation.setResourceId(dataReplicaLocationModel.getResourceId());
+ dataReplicaLocation.setReplicaName(dataReplicaLocationModel.getReplicaName());
+ dataReplicaLocation.setReplicaDescription(dataReplicaLocationModel.getReplicaDescription());
+ dataReplicaLocation.setDataLocations(StringUtils.join(dataReplicaLocationModel.getDataLocations(), ','));
+ dataReplicaLocation.setCreationTime(new Timestamp(dataReplicaLocationModel.getCreationTime()));
+ dataReplicaLocation.setLastModifiedTime(new Timestamp(dataReplicaLocationModel.getLastModifiedTime()));
+ return dataReplicaLocation;
+ }
+ return null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/1a22de16/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/experiment/catalog/impl/RegistryFactory.java
----------------------------------------------------------------------
diff --git a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/experiment/catalog/impl/RegistryFactory.java b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/experiment/catalog/impl/RegistryFactory.java
index c924149..808e6dc 100644
--- a/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/experiment/catalog/impl/RegistryFactory.java
+++ b/modules/registry/registry-core/src/main/java/org/apache/airavata/registry/core/experiment/catalog/impl/RegistryFactory.java
@@ -24,6 +24,7 @@ package org.apache.airavata.registry.core.experiment.catalog.impl;
import org.apache.airavata.common.exception.ApplicationSettingsException;
import org.apache.airavata.common.utils.ServerSettings;
import org.apache.airavata.registry.core.app.catalog.impl.AppCatalogImpl;
+import org.apache.airavata.registry.core.data.catalog.impl.DataCatalogImpl;
import org.apache.airavata.registry.core.impl.RegistryImpl;
import org.apache.airavata.registry.cpi.*;
import org.slf4j.Logger;
@@ -32,6 +33,7 @@ import org.slf4j.LoggerFactory;
public class RegistryFactory {
private static ExperimentCatalog experimentCatalog;
private static AppCatalog appCatalog;
+ private static DataCatalog dataCatalog;
private static Registry registry;
private static Logger logger = LoggerFactory.getLogger(RegistryFactory.class);
@@ -99,6 +101,19 @@ public class RegistryFactory {
}
+ public static DataCatalog getDataCatalog() throws DataCatalogException {
+ try {
+ if (dataCatalog == null) {
+ dataCatalog = new DataCatalogImpl();
+ }
+ } catch (Exception e) {
+ logger.error("Unable to create data catalog instance", e);
+ throw new DataCatalogException(e);
+ }
+ return dataCatalog;
+ }
+
+
public static ExperimentCatalog getLoggingRegistry() {
if(experimentCatalog == null) {
experimentCatalog = new LoggingExperimentCatalogImpl();
http://git-wip-us.apache.org/repos/asf/airavata/blob/1a22de16/modules/registry/registry-core/src/main/resources/META-INF/persistence.xml
----------------------------------------------------------------------
diff --git a/modules/registry/registry-core/src/main/resources/META-INF/persistence.xml b/modules/registry/registry-core/src/main/resources/META-INF/persistence.xml
index 0a5d09e..fc93d1a 100644
--- a/modules/registry/registry-core/src/main/resources/META-INF/persistence.xml
+++ b/modules/registry/registry-core/src/main/resources/META-INF/persistence.xml
@@ -20,7 +20,7 @@
*
* -->
<persistence xmlns="http://java.sun.com/xml/ns/persistence"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.0">
+ version="1.0">
<persistence-unit name="appcatalog_data">
<provider>org.apache.openjpa.persistence.PersistenceProviderImpl</provider>
<class>org.apache.airavata.registry.core.app.catalog.model.ComputeResource</class>
@@ -94,4 +94,10 @@
<class>org.apache.airavata.registry.core.experiment.catalog.model.JobStatus</class>
<exclude-unlisted-classes>true</exclude-unlisted-classes>
</persistence-unit>
+ <persistence-unit name="datacatalog_data">
+ <provider>org.apache.openjpa.persistence.PersistenceProviderImpl</provider>
+ <class>org.apache.airavata.registry.core.data.catalog.model.DataResource</class>
+ <class>org.apache.airavata.registry.core.data.catalog.model.DataReplicaLocation</class>
+ <class>org.apache.airavata.registry.core.data.catalog.model.Configuration</class>
+ </persistence-unit>
</persistence>
http://git-wip-us.apache.org/repos/asf/airavata/blob/1a22de16/modules/registry/registry-core/src/test/java/org/apache/airavata/data/catalog/DataCatalogTest.java
----------------------------------------------------------------------
diff --git a/modules/registry/registry-core/src/test/java/org/apache/airavata/data/catalog/DataCatalogTest.java b/modules/registry/registry-core/src/test/java/org/apache/airavata/data/catalog/DataCatalogTest.java
new file mode 100644
index 0000000..1eb1a25
--- /dev/null
+++ b/modules/registry/registry-core/src/test/java/org/apache/airavata/data/catalog/DataCatalogTest.java
@@ -0,0 +1,133 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.data.catalog;
+
+import org.apache.airavata.data.catalog.util.Initialize;
+import org.apache.airavata.model.data.resource.DataReplicaLocationModel;
+import org.apache.airavata.model.data.resource.DataResourceModel;
+import org.apache.airavata.registry.core.experiment.catalog.impl.RegistryFactory;
+import org.apache.airavata.registry.cpi.DataCatalog;
+import org.apache.airavata.registry.cpi.DataCatalogException;
+import org.junit.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+
+public class DataCatalogTest {
+ private final static Logger logger = LoggerFactory.getLogger(DataCatalogTest.class);
+ private static Initialize initialize;
+ private static DataCatalog datacatalog;
+ private static DataResourceModel dataResourceModel;
+
+ @BeforeClass
+ public static void setUp() {
+ try {
+ System.out.println("********** SET UP ************");
+ initialize = new Initialize("datacatalog-derby.sql");
+ initialize.initializeDB();
+ datacatalog = RegistryFactory.getDataCatalog();
+ dataResourceModel = new DataResourceModel();
+ dataResourceModel.setResourceName("test-file.txt");
+ List<DataReplicaLocationModel> replicaLocationModelList = new ArrayList<>();
+ DataReplicaLocationModel dataReplicaLocationModel = new DataReplicaLocationModel();
+ dataReplicaLocationModel.setReplicaName("1-st-replica");
+ ArrayList<String> dataLocations = new ArrayList<>();
+ dataLocations.add("scp://g75.iu.xsede.org:/var/www/portal/experimentData/test-file.txt");
+ dataReplicaLocationModel.setDataLocations(dataLocations);
+ replicaLocationModelList.add(dataReplicaLocationModel);
+ dataResourceModel.setReplicaLocations(replicaLocationModelList);
+ } catch (DataCatalogException e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ System.out.println("********** TEAR DOWN ************");
+ initialize.stopDerbyServer();
+ }
+
+ @Test
+ public void testPublishDataResource(){
+ try {
+ String resourceId = datacatalog.publishResource(dataResourceModel);
+ org.junit.Assert.assertNotNull(resourceId);
+ } catch (DataCatalogException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void testRemoveDataResource(){
+ try {
+ boolean result = datacatalog.removeResource("234234234");
+ Assert.assertFalse(result);
+ String resourceId = datacatalog.publishResource(dataResourceModel);
+ Assert.assertNotNull(resourceId);
+ result = datacatalog.removeResource(resourceId);
+ Assert.assertTrue(result);
+ result = datacatalog.removeResource(resourceId);
+ Assert.assertFalse(result);
+ } catch (DataCatalogException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void testGetDataResource(){
+ try {
+ String resourceId = datacatalog.publishResource(dataResourceModel);
+ Assert.assertNotNull(resourceId);
+ DataResourceModel persistedCopy = datacatalog.getResource(resourceId);
+ Assert.assertNotNull(persistedCopy);
+ Assert.assertTrue(persistedCopy.getReplicaLocations().size()==1);
+ } catch (DataCatalogException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void testUpdateDataResource(){
+ try {
+ dataResourceModel.setResourceId(UUID.randomUUID().toString());
+ boolean result = datacatalog.updateResource(dataResourceModel);
+ Assert.assertFalse(result);
+ datacatalog.publishResource(dataResourceModel);
+ DataReplicaLocationModel dataReplicaLocationModel = new DataReplicaLocationModel();
+ dataReplicaLocationModel.setReplicaName("2-nd-replica");
+ ArrayList<String> dataLocations = new ArrayList<>();
+ dataLocations.add("scp://g175.iu.xsede.org:/var/www/portal/experimentData/test-file.txt");
+ dataResourceModel.getReplicaLocations().add(dataReplicaLocationModel);
+ datacatalog.updateResource(dataResourceModel);
+ dataResourceModel = datacatalog.getResource(dataResourceModel.getResourceId());
+ Assert.assertTrue(dataResourceModel.getReplicaLocations().size()==2);
+ dataResourceModel.getReplicaLocations().remove(1);
+ datacatalog.updateResource(dataResourceModel);
+ dataResourceModel = datacatalog.getResource(dataResourceModel.getResourceId());
+ Assert.assertTrue(dataResourceModel.getReplicaLocations().size()==1);
+ } catch (DataCatalogException e) {
+ e.printStackTrace();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/1a22de16/modules/registry/registry-core/src/test/java/org/apache/airavata/data/catalog/util/Initialize.java
----------------------------------------------------------------------
diff --git a/modules/registry/registry-core/src/test/java/org/apache/airavata/data/catalog/util/Initialize.java b/modules/registry/registry-core/src/test/java/org/apache/airavata/data/catalog/util/Initialize.java
new file mode 100644
index 0000000..002e51e
--- /dev/null
+++ b/modules/registry/registry-core/src/test/java/org/apache/airavata/data/catalog/util/Initialize.java
@@ -0,0 +1,315 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.data.catalog.util;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.registry.core.data.catalog.utils.DataCatalogConstants;
+import org.apache.derby.drda.NetworkServerControl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.InetAddress;
+import java.net.URI;
+import java.sql.*;
+import java.util.StringTokenizer;
+
+public class Initialize {
+ private static final Logger logger = LoggerFactory.getLogger(Initialize.class);
+ public static final String DERBY_SERVER_MODE_SYS_PROPERTY = "derby.drda.startNetworkServer";
+ public String scriptName = "datacatalog-derby.sql";
+ private NetworkServerControl server;
+ private static final String delimiter = ";";
+ private String jdbcUrl = null;
+ private String jdbcDriver = null;
+ private String jdbcUser = null;
+ private String jdbcPassword = null;
+
+ public Initialize(String scriptName) {
+ this.scriptName = scriptName;
+ }
+
+ public static boolean checkStringBufferEndsWith(StringBuffer buffer, String suffix) {
+ if (suffix.length() > buffer.length()) {
+ return false;
+ }
+ // this loop is done on purpose to avoid memory allocation performance
+ // problems on various JDKs
+ // StringBuffer.lastIndexOf() was introduced in jdk 1.4 and
+ // implementation is ok though does allocation/copying
+ // StringBuffer.toString().endsWith() does massive memory
+ // allocation/copying on JDK 1.5
+ // See http://issues.apache.org/bugzilla/show_bug.cgi?id=37169
+ int endIndex = suffix.length() - 1;
+ int bufferIndex = buffer.length() - 1;
+ while (endIndex >= 0) {
+ if (buffer.charAt(bufferIndex) != suffix.charAt(endIndex)) {
+ return false;
+ }
+ bufferIndex--;
+ endIndex--;
+ }
+ return true;
+ }
+
+ private static boolean isServerStarted(NetworkServerControl server, int ntries)
+ {
+ for (int i = 1; i <= ntries; i ++)
+ {
+ try {
+ Thread.sleep(500);
+ server.ping();
+ return true;
+ }
+ catch (Exception e) {
+ if (i == ntries)
+ return false;
+ }
+ }
+ return false;
+ }
+
+ public void initializeDB() {
+ try{
+ jdbcDriver = ServerSettings.getSetting("datacatalog.jdbc.driver");
+ jdbcUrl = ServerSettings.getSetting("datacatalog.jdbc.url");
+ jdbcUser = ServerSettings.getSetting("datacatalog.jdbc.user");
+ jdbcPassword = ServerSettings.getSetting("datacatalog.jdbc.password");
+ jdbcUrl = jdbcUrl + "?" + "user=" + jdbcUser + "&" + "password=" + jdbcPassword;
+ } catch (ApplicationSettingsException e) {
+ logger.error("Unable to read properties", e);
+ }
+
+ startDerbyInServerMode();
+ if(!isServerStarted(server, 20)){
+ throw new RuntimeException("Derby server could not started within five seconds...");
+ }
+ Connection conn = null;
+ try {
+ Class.forName(jdbcDriver).newInstance();
+ conn = DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPassword);
+ if (!isDatabaseStructureCreated(DataCatalogConstants.CONFIGURATION, conn)) {
+ executeSQLScript(conn);
+ logger.info("New Database created for Data Catalog !!!");
+ } else {
+ logger.debug("Database already created for Data Catalog!");
+ }
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ throw new RuntimeException("Database failure", e);
+ } finally {
+ try {
+ if (conn != null){
+ if (!conn.getAutoCommit()) {
+ conn.commit();
+ }
+ conn.close();
+ }
+ } catch (SQLException e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+ }
+
+ public static boolean isDatabaseStructureCreated(String tableName, Connection conn) {
+ try {
+ System.out.println("Running a query to test the database tables existence.");
+ // check whether the tables are already created with a query
+ Statement statement = null;
+ try {
+ statement = conn.createStatement();
+ ResultSet rs = statement.executeQuery("select * from " + tableName);
+ if (rs != null) {
+ rs.close();
+ }
+ } finally {
+ try {
+ if (statement != null) {
+ statement.close();
+ }
+ } catch (SQLException e) {
+ return false;
+ }
+ }
+ } catch (SQLException e) {
+ return false;
+ }
+
+ return true;
+ }
+
+ private void executeSQLScript(Connection conn) throws Exception {
+ StringBuffer sql = new StringBuffer();
+ BufferedReader reader = null;
+ try{
+
+ InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(scriptName);
+ reader = new BufferedReader(new InputStreamReader(inputStream));
+ String line;
+ while ((line = reader.readLine()) != null) {
+ line = line.trim();
+ if (line.startsWith("//")) {
+ continue;
+ }
+ if (line.startsWith("--")) {
+ continue;
+ }
+ StringTokenizer st = new StringTokenizer(line);
+ if (st.hasMoreTokens()) {
+ String token = st.nextToken();
+ if ("REM".equalsIgnoreCase(token)) {
+ continue;
+ }
+ }
+ sql.append(" ").append(line);
+
+ // SQL defines "--" as a comment to EOL
+ // and in Oracle it may contain a hint
+ // so we cannot just remove it, instead we must end it
+ if (line.indexOf("--") >= 0) {
+ sql.append("\n");
+ }
+ if ((checkStringBufferEndsWith(sql, delimiter))) {
+ executeSQL(sql.substring(0, sql.length() - delimiter.length()), conn);
+ sql.replace(0, sql.length(), "");
+ }
+ }
+ // Catch any statements not followed by ;
+ if (sql.length() > 0) {
+ executeSQL(sql.toString(), conn);
+ }
+ }catch (IOException e){
+ logger.error("Error occurred while executing SQL script for creating Airavata Data Catalog database", e);
+ throw new Exception("Error occurred while executing SQL script for creating Airavata Data Catalog database", e);
+ }finally {
+ if (reader != null) {
+ reader.close();
+ }
+ }
+ }
+
+ private static void executeSQL(String sql, Connection conn) throws Exception {
+ // Check and ignore empty statements
+ if ("".equals(sql.trim())) {
+ return;
+ }
+
+ Statement statement = null;
+ try {
+ logger.debug("SQL : " + sql);
+
+ boolean ret;
+ int updateCount = 0, updateCountTotal = 0;
+ statement = conn.createStatement();
+ ret = statement.execute(sql);
+ updateCount = statement.getUpdateCount();
+ do {
+ if (!ret) {
+ if (updateCount != -1) {
+ updateCountTotal += updateCount;
+ }
+ }
+ ret = statement.getMoreResults();
+ if (ret) {
+ updateCount = statement.getUpdateCount();
+ }
+ } while (ret);
+
+ logger.debug(sql + " : " + updateCountTotal + " rows affected");
+
+ SQLWarning warning = conn.getWarnings();
+ while (warning != null) {
+ logger.warn(warning + " sql warning");
+ warning = warning.getNextWarning();
+ }
+ conn.clearWarnings();
+ } catch (SQLException e) {
+ if (e.getSQLState().equals("X0Y32")) {
+ // eliminating the table already exception for the derby
+ // database
+ logger.info("Table Already Exists", e);
+ } else {
+ throw new Exception("Error occurred while executing : " + sql, e);
+ }
+ } finally {
+ if (statement != null) {
+ try {
+ statement.close();
+ } catch (SQLException e) {
+ logger.error("Error occurred while closing result set.", e);
+ }
+ }
+ }
+ }
+
+ private void startDerbyInServerMode() {
+ try {
+ System.setProperty(DERBY_SERVER_MODE_SYS_PROPERTY, "true");
+ server = new NetworkServerControl(InetAddress.getByName("0.0.0.0"),
+ 20000,
+ jdbcUser, jdbcPassword);
+ java.io.PrintWriter consoleWriter = new java.io.PrintWriter(System.out, true);
+ server.start(consoleWriter);
+ } catch (IOException e) {
+ logger.error("Unable to start Apache derby in the server mode! Check whether " +
+ "specified port is available");
+ } catch (Exception e) {
+ logger.error("Unable to start Apache derby in the server mode! Check whether " +
+ "specified port is available");
+ }
+
+ }
+
+ public static int getPort(String jdbcURL){
+ try{
+ String cleanURI = jdbcURL.substring(5);
+ URI uri = URI.create(cleanURI);
+ return uri.getPort();
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ return -1;
+ }
+ }
+
+ private void startDerbyInEmbeddedMode(){
+ try {
+ Class.forName("org.apache.derby.jdbc.EmbeddedDriver");
+ DriverManager.getConnection("jdbc:derby:memory:unit-testing-jpa;create=true").close();
+ } catch (ClassNotFoundException e) {
+ logger.error(e.getMessage(), e);
+ } catch (SQLException e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+
+ public void stopDerbyServer() {
+ try {
+ server.shutdown();
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/1a22de16/modules/registry/registry-core/src/test/resources/datacatalog-derby.sql
----------------------------------------------------------------------
diff --git a/modules/registry/registry-core/src/test/resources/datacatalog-derby.sql b/modules/registry/registry-core/src/test/resources/datacatalog-derby.sql
new file mode 100644
index 0000000..3091447
--- /dev/null
+++ b/modules/registry/registry-core/src/test/resources/datacatalog-derby.sql
@@ -0,0 +1,52 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+CREATE TABLE DATA_RESOURCE
+(
+ RESOURCE_ID VARCHAR (255),
+ RESOURCE_NAME VARCHAR (255),
+ RESOURCE_DESCRIPTION VARCHAR (1024),
+ RESOURCE_SIZE INTEGER ,
+ CREATION_TIME TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+ LAST_MODIFIED_TIME TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+ PRIMARY KEY (RESOURCE_ID)
+);
+
+CREATE TABLE DATA_REPLICA_LOCATION
+(
+ RESOURCE_ID VARCHAR (255),
+ REPLICA_NAME VARCHAR (255),
+ REPLICA_DESCRIPTION VARCHAR (1024),
+ DATA_LOCATIONS VARCHAR (4096),
+ CREATION_TIME TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+ LAST_MODIFIED_TIME TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+ FOREIGN KEY (RESOURCE_ID) REFERENCES DATA_RESOURCE(RESOURCE_ID) ON DELETE CASCADE
+);
+
+
+CREATE TABLE CONFIGURATION
+(
+ CONFIG_KEY VARCHAR(255),
+ CONFIG_VAL VARCHAR(255),
+ PRIMARY KEY(CONFIG_KEY, CONFIG_VAL)
+);
+
+INSERT INTO CONFIGURATION (CONFIG_KEY, CONFIG_VAL) VALUES('data_catalog_version', '0.16');
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/1a22de16/modules/registry/registry-cpi/src/main/java/org/apache/airavata/registry/cpi/DataCatalog.java
----------------------------------------------------------------------
diff --git a/modules/registry/registry-cpi/src/main/java/org/apache/airavata/registry/cpi/DataCatalog.java b/modules/registry/registry-cpi/src/main/java/org/apache/airavata/registry/cpi/DataCatalog.java
new file mode 100644
index 0000000..c2d3756
--- /dev/null
+++ b/modules/registry/registry-cpi/src/main/java/org/apache/airavata/registry/cpi/DataCatalog.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.airavata.registry.cpi;
+
+import org.apache.airavata.model.data.resource.*;
+
+public interface DataCatalog {
+
+ String publishResource(DataResourceModel resource) throws DataCatalogException;
+
+ boolean removeResource(String resourceId) throws DataCatalogException;
+
+ boolean updateResource(DataResourceModel resource) throws DataCatalogException;
+
+ DataResourceModel getResource(String resourceId) throws DataCatalogException;
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/1a22de16/modules/registry/registry-cpi/src/main/java/org/apache/airavata/registry/cpi/DataCatalogException.java
----------------------------------------------------------------------
diff --git a/modules/registry/registry-cpi/src/main/java/org/apache/airavata/registry/cpi/DataCatalogException.java b/modules/registry/registry-cpi/src/main/java/org/apache/airavata/registry/cpi/DataCatalogException.java
new file mode 100644
index 0000000..dbb0ee4
--- /dev/null
+++ b/modules/registry/registry-cpi/src/main/java/org/apache/airavata/registry/cpi/DataCatalogException.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.airavata.registry.cpi;
+
+public class DataCatalogException extends Exception{
+ private static final long serialVersionUID = -2849422320739447602L;
+
+ public DataCatalogException(Throwable e) {
+ super(e);
+ }
+
+ public DataCatalogException(String message) {
+ super(message, null);
+ }
+
+ public DataCatalogException(String message, Throwable e) {
+ super(message, e);
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/1a22de16/thrift-interface-descriptions/airavata-api/airavata_api.thrift
----------------------------------------------------------------------
diff --git a/thrift-interface-descriptions/airavata-api/airavata_api.thrift b/thrift-interface-descriptions/airavata-api/airavata_api.thrift
index 7dbecf7..a2b7abd 100644
--- a/thrift-interface-descriptions/airavata-api/airavata_api.thrift
+++ b/thrift-interface-descriptions/airavata-api/airavata_api.thrift
@@ -2323,7 +2323,8 @@ service Airavata {
4: airavata_errors.AuthorizationException ae)
- void publishDataResource(1: required security_model.AuthzToken authzToken, 2: data_resource_models.ResourceModel resourceModel)
+ //Data Manager API methods
+ string publishDataResource(1: required security_model.AuthzToken authzToken, 2: data_resource_models.DataResourceModel resourceModel)
throws (1: airavata_errors.InvalidRequestException ire,
2: airavata_errors.AiravataClientException ace,
3: airavata_errors.AiravataSystemException ase,
http://git-wip-us.apache.org/repos/asf/airavata/blob/1a22de16/thrift-interface-descriptions/airavata-api/data_resource_models.thrift
----------------------------------------------------------------------
diff --git a/thrift-interface-descriptions/airavata-api/data_resource_models.thrift b/thrift-interface-descriptions/airavata-api/data_resource_models.thrift
index 85a8044..4130eca 100644
--- a/thrift-interface-descriptions/airavata-api/data_resource_models.thrift
+++ b/thrift-interface-descriptions/airavata-api/data_resource_models.thrift
@@ -23,13 +23,23 @@
namespace cpp apache.airavata.model.data.resource
namespace py apache.airavata.model.data.resource
-struct ResourceModel {
+struct DataResourceModel {
1: optional string resourceId,
2: optional string resourceName,
- 3: list<ReplicaLocationModel> replicaLocations
+ 3: optional string resourceDescription,
+ 4: optional i32 resourceSize,
+ 5: optional i64 creationTime,
+ 6: optional i64 lastModifiedTime,
+ 7: list<DataReplicaLocationModel> replicaLocations
}
-struct ReplicaLocationModel {
- 1: optional list<string> physicalLocations
+struct DataReplicaLocationModel {
+ 1: optional string replicaId,
+ 2: optional string resourceId,
+ 3: optional string replicaName,
+ 4: optional string replicaDescription,
+ 5: optional i64 creationTime,
+ 6: optional i64 lastModifiedTime,
+ 7: optional list<string> dataLocations
}