You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2023/02/17 14:19:52 UTC
[airavata-data-catalog] 02/05: Implement adding/removing data product to/from metadata schema
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch machristie/issue7
in repository https://gitbox.apache.org/repos/asf/airavata-data-catalog.git
commit 584cff2099e105e4fd99549ac01799e058bf4fae
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Wed Feb 15 15:31:19 2023 -0500
Implement adding/removing data product to/from metadata schema
---
.../api/client/DataCatalogAPIClient.java | 65 ++++++++++++++++++++--
.../datacatalog/api/mapper/DataProductMapper.java | 6 ++
.../datacatalog/api/model/DataProductEntity.java | 24 ++++++++
.../api/repository/DataProductRepository.java | 1 +
.../api/service/DataCatalogAPIService.java | 53 ++++++++++++++++--
.../stubs/src/main/proto/DataCatalogAPI.proto | 9 +++
6 files changed, 146 insertions(+), 12 deletions(-)
diff --git a/data-catalog-api/client/src/main/java/org/apache/airavata/datacatalog/api/client/DataCatalogAPIClient.java b/data-catalog-api/client/src/main/java/org/apache/airavata/datacatalog/api/client/DataCatalogAPIClient.java
index 954f994..3d59fdb 100644
--- a/data-catalog-api/client/src/main/java/org/apache/airavata/datacatalog/api/client/DataCatalogAPIClient.java
+++ b/data-catalog-api/client/src/main/java/org/apache/airavata/datacatalog/api/client/DataCatalogAPIClient.java
@@ -7,11 +7,15 @@ import java.util.concurrent.TimeUnit;
import org.apache.airavata.datacatalog.api.DataCatalogAPIServiceGrpc;
import org.apache.airavata.datacatalog.api.DataCatalogAPIServiceGrpc.DataCatalogAPIServiceBlockingStub;
import org.apache.airavata.datacatalog.api.DataProduct;
+import org.apache.airavata.datacatalog.api.DataProductAddToMetadataSchemaRequest;
+import org.apache.airavata.datacatalog.api.DataProductAddToMetadataSchemaResponse;
import org.apache.airavata.datacatalog.api.DataProductCreateRequest;
import org.apache.airavata.datacatalog.api.DataProductCreateResponse;
import org.apache.airavata.datacatalog.api.DataProductDeleteRequest;
import org.apache.airavata.datacatalog.api.DataProductGetRequest;
import org.apache.airavata.datacatalog.api.DataProductGetResponse;
+import org.apache.airavata.datacatalog.api.DataProductRemoveFromMetadataSchemaRequest;
+import org.apache.airavata.datacatalog.api.DataProductRemoveFromMetadataSchemaResponse;
import org.apache.airavata.datacatalog.api.DataProductUpdateRequest;
import org.apache.airavata.datacatalog.api.DataProductUpdateResponse;
import org.apache.airavata.datacatalog.api.FieldValueType;
@@ -23,10 +27,14 @@ import org.apache.airavata.datacatalog.api.MetadataSchemaFieldCreateRequest;
import org.apache.airavata.datacatalog.api.MetadataSchemaFieldCreateResponse;
import org.apache.airavata.datacatalog.api.MetadataSchemaFieldListRequest;
import org.apache.airavata.datacatalog.api.MetadataSchemaFieldListResponse;
+import org.apache.airavata.datacatalog.api.MetadataSchemaGetRequest;
+import org.apache.airavata.datacatalog.api.MetadataSchemaGetResponse;
import io.grpc.Channel;
import io.grpc.ManagedChannel;
-import io.grpc.ManagedChannelBuilder;;
+import io.grpc.ManagedChannelBuilder;
+import io.grpc.Status;
+import io.grpc.StatusRuntimeException;;
public class DataCatalogAPIClient {
@@ -67,6 +75,19 @@ public class DataCatalogAPIClient {
return response.getMetadataSchema();
}
+ public MetadataSchema getMetadataSchema(String schemaName) {
+ MetadataSchemaGetRequest request = MetadataSchemaGetRequest.newBuilder().setSchemaName(schemaName).build();
+ try {
+ MetadataSchemaGetResponse response = blockingStub.getMetadataSchema(request);
+ return response.getMetadataSchema();
+ } catch (StatusRuntimeException e) {
+ if (e.getStatus() == Status.NOT_FOUND) {
+ return null;
+ }
+ throw e;
+ }
+ }
+
public MetadataSchemaField createMetadataSchemaField(MetadataSchemaField metadataSchemaField) {
MetadataSchemaFieldCreateRequest request = MetadataSchemaFieldCreateRequest.newBuilder()
.setMetadataSchemaField(metadataSchemaField).build();
@@ -81,6 +102,21 @@ public class DataCatalogAPIClient {
return response.getMetadataSchemaFieldsList();
}
+ public DataProduct addDataProductToMetadataSchema(String dataProductId, String schemaName) {
+ DataProductAddToMetadataSchemaRequest request = DataProductAddToMetadataSchemaRequest.newBuilder()
+ .setDataProductId(dataProductId).setSchemaName(schemaName).build();
+ DataProductAddToMetadataSchemaResponse response = blockingStub.addDataProductToMetadataSchema(request);
+ return response.getDataProduct();
+ }
+
+ public DataProduct removeDataProductFromMetadataSchema(String dataProductId, String schemaName) {
+ DataProductRemoveFromMetadataSchemaRequest request = DataProductRemoveFromMetadataSchemaRequest.newBuilder()
+ .setDataProductId(dataProductId).setSchemaName(schemaName).build();
+ DataProductRemoveFromMetadataSchemaResponse response = blockingStub
+ .removeDataProductFromMetadataSchema(request);
+ return response.getDataProduct();
+ }
+
public static void main(String[] args) throws InterruptedException {
String target = "localhost:6565";
@@ -115,11 +151,19 @@ public class DataCatalogAPIClient {
System.out.println(
MessageFormat.format("Deleted data product with id [{0}]", result2.getDataProductId()));
- MetadataSchema metadataSchema = MetadataSchema.newBuilder().setSchemaName("my_schema").build();
- metadataSchema = client.createMetadataSchema(metadataSchema);
-
- System.out.println(
- MessageFormat.format("Created metadata schema with name [{0}]", metadataSchema.getSchemaName()));
+ // First check if metadata schema exists
+ MetadataSchema metadataSchema = client.getMetadataSchema("my_schema");
+ if (metadataSchema == null) {
+ metadataSchema = MetadataSchema.newBuilder().setSchemaName("my_schema").build();
+ metadataSchema = client.createMetadataSchema(metadataSchema);
+ System.out.println(
+ MessageFormat.format("Created metadata schema with name [{0}]",
+ metadataSchema.getSchemaName()));
+ } else {
+ System.out.println(
+ MessageFormat.format("Found metadata schema with name [{0}]",
+ metadataSchema.getSchemaName()));
+ }
MetadataSchemaField field1 = MetadataSchemaField.newBuilder().setFieldName("field1")
.setJsonPath("$.field1").setValueType(FieldValueType.FLOAT)
@@ -142,6 +186,15 @@ public class DataCatalogAPIClient {
System.out.println(MessageFormat.format("-> field {0}", field.getFieldName()));
}
+ result = client.addDataProductToMetadataSchema(result.getDataProductId(), metadataSchema.getSchemaName());
+ System.out.println(MessageFormat.format("Added data product [{0}] to metadata schema [{1}]",
+ result.getDataProductId(), metadataSchema.getSchemaName()));
+
+ result = client.removeDataProductFromMetadataSchema(result.getDataProductId(),
+ metadataSchema.getSchemaName());
+ System.out.println(MessageFormat.format("Removed data product [{0}] from metadata schema [{1}]",
+ result.getDataProductId(), metadataSchema.getSchemaName()));
+
} finally {
channel.shutdownNow().awaitTermination(5, TimeUnit.SECONDS);
}
diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/mapper/DataProductMapper.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/mapper/DataProductMapper.java
index 8e1e155..b03fe1d 100644
--- a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/mapper/DataProductMapper.java
+++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/mapper/DataProductMapper.java
@@ -2,6 +2,7 @@ package org.apache.airavata.datacatalog.api.mapper;
import org.apache.airavata.datacatalog.api.DataProduct;
import org.apache.airavata.datacatalog.api.model.DataProductEntity;
+import org.apache.airavata.datacatalog.api.model.MetadataSchemaEntity;
import org.apache.airavata.datacatalog.api.repository.DataProductRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@@ -50,5 +51,10 @@ public class DataProductMapper {
if (dataProductEntity.getParentDataProductEntity() != null) {
dataProductBuilder.setParentDataProductId(dataProductEntity.getParentDataProductEntity().getExternalId());
}
+ if (dataProductEntity.getMetadataSchemas() != null) {
+ for (MetadataSchemaEntity metadataSchema : dataProductEntity.getMetadataSchemas()) {
+ dataProductBuilder.addSchemaName(metadataSchema.getSchemaName());
+ }
+ }
}
}
diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/model/DataProductEntity.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/model/DataProductEntity.java
index 65b24e7..68b615f 100644
--- a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/model/DataProductEntity.java
+++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/model/DataProductEntity.java
@@ -1,5 +1,7 @@
package org.apache.airavata.datacatalog.api.model;
+import java.util.Set;
+
import org.hibernate.annotations.Type;
import com.fasterxml.jackson.databind.JsonNode;
@@ -12,6 +14,8 @@ import jakarta.persistence.GeneratedValue;
import jakarta.persistence.GenerationType;
import jakarta.persistence.Id;
import jakarta.persistence.JoinColumn;
+import jakarta.persistence.JoinTable;
+import jakarta.persistence.ManyToMany;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.SequenceGenerator;
import jakarta.persistence.Table;
@@ -43,6 +47,10 @@ public class DataProductEntity {
@Column(name = "metadata", columnDefinition = "jsonb")
private JsonNode metadata;
+ @ManyToMany
+ @JoinTable(name = "data_product_metadata_schema", joinColumns = @JoinColumn(name = "data_product_id"), inverseJoinColumns = @JoinColumn(name = "metadata_schema_id"))
+ private Set<MetadataSchemaEntity> metadataSchemas;
+
// TODO: ManyToOne mapping to owner: UserEntity
public Long getDataProductId() {
return dataProductId;
@@ -84,6 +92,22 @@ public class DataProductEntity {
this.metadata = metadata;
}
+ public Set<MetadataSchemaEntity> getMetadataSchemas() {
+ return metadataSchemas;
+ }
+
+ public void setMetadataSchemas(Set<MetadataSchemaEntity> metadataSchemas) {
+ this.metadataSchemas = metadataSchemas;
+ }
+
+ public void addMetadataSchema(MetadataSchemaEntity metadataSchema) {
+ this.metadataSchemas.add(metadataSchema);
+ }
+
+ public void removeMetadataSchema(MetadataSchemaEntity metadataSchema) {
+ this.metadataSchemas.remove(metadataSchema);
+ }
+
@Override
public int hashCode() {
final int prime = 31;
diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/repository/DataProductRepository.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/repository/DataProductRepository.java
index acec036..426e4d4 100644
--- a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/repository/DataProductRepository.java
+++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/repository/DataProductRepository.java
@@ -4,6 +4,7 @@ import org.apache.airavata.datacatalog.api.model.DataProductEntity;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.transaction.annotation.Transactional;
+@Transactional(readOnly = true)
public interface DataProductRepository extends JpaRepository<DataProductEntity, Long> {
DataProductEntity findByExternalId(String externalId);
diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/service/DataCatalogAPIService.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/service/DataCatalogAPIService.java
index 3869ee5..7374def 100644
--- a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/service/DataCatalogAPIService.java
+++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/service/DataCatalogAPIService.java
@@ -28,6 +28,8 @@ import org.apache.airavata.datacatalog.api.MetadataSchemaFieldListRequest;
import org.apache.airavata.datacatalog.api.MetadataSchemaFieldListResponse;
import org.apache.airavata.datacatalog.api.MetadataSchemaFieldUpdateRequest;
import org.apache.airavata.datacatalog.api.MetadataSchemaFieldUpdateResponse;
+import org.apache.airavata.datacatalog.api.MetadataSchemaGetRequest;
+import org.apache.airavata.datacatalog.api.MetadataSchemaGetResponse;
import org.apache.airavata.datacatalog.api.mapper.DataProductMapper;
import org.apache.airavata.datacatalog.api.mapper.MetadataSchemaFieldMapper;
import org.apache.airavata.datacatalog.api.mapper.MetadataSchemaMapper;
@@ -41,10 +43,13 @@ import org.lognet.springboot.grpc.GRpcService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.transaction.annotation.Transactional;
+import io.grpc.Status;
import io.grpc.stub.StreamObserver;
@GRpcService
+@Transactional
public class DataCatalogAPIService extends DataCatalogAPIServiceGrpc.DataCatalogAPIServiceImplBase {
private static final Logger logger = LoggerFactory.getLogger(DataCatalogAPIService.class);
@@ -76,7 +81,7 @@ public class DataCatalogAPIService extends DataCatalogAPIServiceGrpc.DataCatalog
DataProductEntity dataProductEntity = new DataProductEntity();
dataProductEntity.setExternalId(UUID.randomUUID().toString());
dataProductMapper.mapModelToEntity(request.getDataProduct(), dataProductEntity);
- DataProductEntity savedDataProductEntity = dataProductRepository.save(dataProductEntity);
+ DataProductEntity savedDataProductEntity = dataProductRepository.saveAndFlush(dataProductEntity);
// TODO: SharingManager.grantPermissionToUser(userInfo, dataProduct,
// Permission.OWNER)
@@ -131,8 +136,33 @@ public class DataCatalogAPIService extends DataCatalogAPIServiceGrpc.DataCatalog
@Override
public void addDataProductToMetadataSchema(DataProductAddToMetadataSchemaRequest request,
StreamObserver<DataProductAddToMetadataSchemaResponse> responseObserver) {
- // TODO Auto-generated method stub
- super.addDataProductToMetadataSchema(request, responseObserver);
+ String dataProductId = request.getDataProductId();
+ // TODO: handle data product not found
+ DataProductEntity dataProduct = dataProductRepository.findByExternalId(dataProductId);
+ String schemaName = request.getSchemaName();
+ // TODO: handle metadata schema not found
+ MetadataSchemaEntity metadataSchemaEntity = metadataSchemaRepository.findBySchemaName(schemaName);
+ dataProduct.addMetadataSchema(metadataSchemaEntity);
+
+ DataProductAddToMetadataSchemaResponse.Builder responseBuilder = DataProductAddToMetadataSchemaResponse
+ .newBuilder();
+ dataProductMapper.mapEntityToModel(dataProduct, responseBuilder.getDataProductBuilder());
+ responseObserver.onNext(responseBuilder.build());
+ responseObserver.onCompleted();
+ }
+
+ @Override
+ public void getMetadataSchema(MetadataSchemaGetRequest request,
+ StreamObserver<MetadataSchemaGetResponse> responseObserver) {
+ MetadataSchemaEntity metadataSchemaEntity = metadataSchemaRepository.findBySchemaName(request.getSchemaName());
+ if (metadataSchemaEntity == null) {
+ responseObserver.onError(Status.NOT_FOUND.asException());
+ responseObserver.onCompleted();
+ }
+ MetadataSchemaGetResponse.Builder responseBuilder = MetadataSchemaGetResponse.newBuilder();
+ metadataSchemaMapper.mapEntityToModel(metadataSchemaEntity, responseBuilder.getMetadataSchemaBuilder());
+ responseObserver.onNext(responseBuilder.build());
+ responseObserver.onCompleted();
}
@Override
@@ -193,15 +223,26 @@ public class DataCatalogAPIService extends DataCatalogAPIServiceGrpc.DataCatalog
metadataSchemaFieldMapper.mapEntityToModel(metadataSchemaFieldEntity, builder);
}
responseObserver.onNext(responseBuilder.build());
- responseObserver.onError(null);
responseObserver.onCompleted();
}
@Override
public void removeDataProductFromMetadataSchema(DataProductRemoveFromMetadataSchemaRequest request,
StreamObserver<DataProductRemoveFromMetadataSchemaResponse> responseObserver) {
- // TODO Auto-generated method stub
- super.removeDataProductFromMetadataSchema(request, responseObserver);
+
+ String dataProductId = request.getDataProductId();
+ // TODO: handle data product not found
+ DataProductEntity dataProduct = dataProductRepository.findByExternalId(dataProductId);
+ String schemaName = request.getSchemaName();
+ // TODO: handle metadata schema not found
+ MetadataSchemaEntity metadataSchemaEntity = metadataSchemaRepository.findBySchemaName(schemaName);
+ dataProduct.removeMetadataSchema(metadataSchemaEntity);
+
+ DataProductRemoveFromMetadataSchemaResponse.Builder responseBuilder = DataProductRemoveFromMetadataSchemaResponse
+ .newBuilder();
+ dataProductMapper.mapEntityToModel(dataProduct, responseBuilder.getDataProductBuilder());
+ responseObserver.onNext(responseBuilder.build());
+ responseObserver.onCompleted();
}
@Override
diff --git a/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto b/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto
index b1d40dd..cb03016 100644
--- a/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto
+++ b/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto
@@ -62,6 +62,7 @@ message DataProduct {
optional string parent_data_product_id = 2;
string name = 3;
optional string metadata = 4;
+ repeated string schema_name = 5;
}
enum FieldValueType {
@@ -105,6 +106,13 @@ message DataProductDeleteRequest {
message DataProductDeleteResponse {
}
+message MetadataSchemaGetRequest {
+ UserInfo user_info = 1;
+ string schema_name = 2;
+}
+message MetadataSchemaGetResponse {
+ MetadataSchema metadata_schema = 1;
+}
message MetadataSchemaCreateRequest {
UserInfo user_info = 1;
MetadataSchema metadata_schema = 2;
@@ -168,6 +176,7 @@ service DataCatalogAPIService {
rpc updateDataProduct(DataProductUpdateRequest) returns (DataProductUpdateResponse){}
rpc getDataProduct(DataProductGetRequest) returns (DataProductGetResponse){}
rpc deleteDataProduct(DataProductDeleteRequest) returns (DataProductDeleteResponse){}
+ rpc getMetadataSchema(MetadataSchemaGetRequest) returns (MetadataSchemaGetResponse){}
rpc createMetadataSchema(MetadataSchemaCreateRequest) returns (MetadataSchemaCreateResponse){}
rpc deleteMetadataSchema(MetadataSchemaCreateRequest) returns (MetadataSchemaCreateResponse){}
rpc createMetadataSchemaField(MetadataSchemaFieldCreateRequest) returns (MetadataSchemaFieldCreateResponse){}