You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2023/02/17 14:19:50 UTC
[airavata-data-catalog] branch machristie/issue7 created (now 88be4d0)
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a change to branch machristie/issue7
in repository https://gitbox.apache.org/repos/asf/airavata-data-catalog.git
at 88be4d0 Additional methods to manage metadata schemas
This branch includes the following new commits:
new 131ca0e Initial work at implementing metadata schema management APIs
new 584cff2 Implement adding/removing data product to/from metadata schema
new 011dff8 moved test package
new 9d7f93b Support creating data product for known metadata schemas
new 88be4d0 Additional methods to manage metadata schemas
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[airavata-data-catalog] 05/05: Additional methods to manage metadata schemas
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch machristie/issue7
in repository https://gitbox.apache.org/repos/asf/airavata-data-catalog.git
commit 88be4d08bbbb4f233f04f98cb99b7b95bad52bc5
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Wed Feb 15 15:57:25 2023 -0500
Additional methods to manage metadata schemas
---
.../repository/MetadataSchemaFieldRepository.java | 2 +
.../api/service/DataCatalogAPIService.java | 43 ++++++++++++++++++----
.../stubs/src/main/proto/DataCatalogAPI.proto | 2 +-
3 files changed, 38 insertions(+), 9 deletions(-)
diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/repository/MetadataSchemaFieldRepository.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/repository/MetadataSchemaFieldRepository.java
index e237dac..c8503db 100644
--- a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/repository/MetadataSchemaFieldRepository.java
+++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/repository/MetadataSchemaFieldRepository.java
@@ -8,4 +8,6 @@ import org.springframework.data.jpa.repository.JpaRepository;
public interface MetadataSchemaFieldRepository extends JpaRepository<MetadataSchemaFieldEntity, Long> {
List<MetadataSchemaFieldEntity> findByMetadataSchema_SchemaName(String schemaName);
+
+ MetadataSchemaFieldEntity findByFieldNameAndSchema_SchemaName(String fieldName, String schemaName);
}
diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/service/DataCatalogAPIService.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/service/DataCatalogAPIService.java
index 7374def..d001e46 100644
--- a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/service/DataCatalogAPIService.java
+++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/service/DataCatalogAPIService.java
@@ -19,6 +19,8 @@ import org.apache.airavata.datacatalog.api.DataProductUpdateRequest;
import org.apache.airavata.datacatalog.api.DataProductUpdateResponse;
import org.apache.airavata.datacatalog.api.MetadataSchemaCreateRequest;
import org.apache.airavata.datacatalog.api.MetadataSchemaCreateResponse;
+import org.apache.airavata.datacatalog.api.MetadataSchemaDeleteRequest;
+import org.apache.airavata.datacatalog.api.MetadataSchemaDeleteResponse;
import org.apache.airavata.datacatalog.api.MetadataSchemaField;
import org.apache.airavata.datacatalog.api.MetadataSchemaFieldCreateRequest;
import org.apache.airavata.datacatalog.api.MetadataSchemaFieldCreateResponse;
@@ -196,17 +198,30 @@ public class DataCatalogAPIService extends DataCatalogAPIServiceGrpc.DataCatalog
}
@Override
- public void deleteMetadataSchema(MetadataSchemaCreateRequest request,
- StreamObserver<MetadataSchemaCreateResponse> responseObserver) {
- // TODO Auto-generated method stub
- super.deleteMetadataSchema(request, responseObserver);
+ public void deleteMetadataSchema(MetadataSchemaDeleteRequest request,
+ StreamObserver<MetadataSchemaDeleteResponse> responseObserver) {
+ // TODO: check that user has write access on metadata schema
+ // TODO: handle metadata schema not found
+ MetadataSchemaEntity metadataSchemaEntity = metadataSchemaRepository
+ .findBySchemaName(request.getMetadataSchema().getSchemaName());
+ metadataSchemaRepository.delete(metadataSchemaEntity);
+ MetadataSchemaDeleteResponse.Builder responseBuilder = MetadataSchemaDeleteResponse.newBuilder();
+ responseObserver.onNext(responseBuilder.build());
+ responseObserver.onCompleted();
}
@Override
public void deleteMetadataSchemaField(MetadataSchemaFieldDeleteRequest request,
StreamObserver<MetadataSchemaFieldDeleteResponse> responseObserver) {
- // TODO Auto-generated method stub
- super.deleteMetadataSchemaField(request, responseObserver);
+ // TODO: check that user has write access on metadata schema field
+ // TODO: handle metadata schema field not found
+ MetadataSchemaFieldEntity metadataSchemaFieldEntity = metadataSchemaFieldRepository
+ .findByFieldNameAndSchema_SchemaName(request.getMetadataSchemaField().getFieldName(),
+ request.getMetadataSchemaField().getSchemaName());
+ metadataSchemaFieldRepository.delete(metadataSchemaFieldEntity);
+ MetadataSchemaFieldDeleteResponse.Builder responseBuilder = MetadataSchemaFieldDeleteResponse.newBuilder();
+ responseObserver.onNext(responseBuilder.build());
+ responseObserver.onCompleted();
}
@Override
@@ -248,7 +263,19 @@ public class DataCatalogAPIService extends DataCatalogAPIServiceGrpc.DataCatalog
@Override
public void updateMetadataSchemaField(MetadataSchemaFieldUpdateRequest request,
StreamObserver<MetadataSchemaFieldUpdateResponse> responseObserver) {
- // TODO Auto-generated method stub
- super.updateMetadataSchemaField(request, responseObserver);
+
+ // TODO: check that user has write access on metadata schema field
+ // TODO: handle metadata schema field not found
+ MetadataSchemaFieldEntity metadataSchemaFieldEntity = metadataSchemaFieldRepository
+ .findByFieldNameAndSchema_SchemaName(request.getMetadataSchemaField().getFieldName(),
+ request.getMetadataSchemaField().getSchemaName());
+ metadataSchemaFieldMapper.mapModelToEntity(request.getMetadataSchemaField(), metadataSchemaFieldEntity);
+ metadataSchemaFieldRepository.save(metadataSchemaFieldEntity);
+
+ MetadataSchemaFieldUpdateResponse.Builder responseBuilder = MetadataSchemaFieldUpdateResponse.newBuilder();
+ metadataSchemaFieldMapper.mapEntityToModel(metadataSchemaFieldEntity,
+ responseBuilder.getMetadataSchemaFieldBuilder());
+ responseObserver.onNext(responseBuilder.build());
+ responseObserver.onCompleted();
}
}
diff --git a/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto b/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto
index 7d91b8b..74261d1 100644
--- a/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto
+++ b/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto
@@ -178,7 +178,7 @@ service DataCatalogAPIService {
rpc deleteDataProduct(DataProductDeleteRequest) returns (DataProductDeleteResponse){}
rpc getMetadataSchema(MetadataSchemaGetRequest) returns (MetadataSchemaGetResponse){}
rpc createMetadataSchema(MetadataSchemaCreateRequest) returns (MetadataSchemaCreateResponse){}
- rpc deleteMetadataSchema(MetadataSchemaCreateRequest) returns (MetadataSchemaCreateResponse){}
+ rpc deleteMetadataSchema(MetadataSchemaDeleteRequest) returns (MetadataSchemaDeleteResponse){}
rpc createMetadataSchemaField(MetadataSchemaFieldCreateRequest) returns (MetadataSchemaFieldCreateResponse){}
rpc updateMetadataSchemaField(MetadataSchemaFieldUpdateRequest) returns (MetadataSchemaFieldUpdateResponse){}
rpc deleteMetadataSchemaField(MetadataSchemaFieldDeleteRequest) returns (MetadataSchemaFieldDeleteResponse){}
[airavata-data-catalog] 02/05: Implement adding/removing data product to/from metadata schema
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch machristie/issue7
in repository https://gitbox.apache.org/repos/asf/airavata-data-catalog.git
commit 584cff2099e105e4fd99549ac01799e058bf4fae
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Wed Feb 15 15:31:19 2023 -0500
Implement adding/removing data product to/from metadata schema
---
.../api/client/DataCatalogAPIClient.java | 65 ++++++++++++++++++++--
.../datacatalog/api/mapper/DataProductMapper.java | 6 ++
.../datacatalog/api/model/DataProductEntity.java | 24 ++++++++
.../api/repository/DataProductRepository.java | 1 +
.../api/service/DataCatalogAPIService.java | 53 ++++++++++++++++--
.../stubs/src/main/proto/DataCatalogAPI.proto | 9 +++
6 files changed, 146 insertions(+), 12 deletions(-)
diff --git a/data-catalog-api/client/src/main/java/org/apache/airavata/datacatalog/api/client/DataCatalogAPIClient.java b/data-catalog-api/client/src/main/java/org/apache/airavata/datacatalog/api/client/DataCatalogAPIClient.java
index 954f994..3d59fdb 100644
--- a/data-catalog-api/client/src/main/java/org/apache/airavata/datacatalog/api/client/DataCatalogAPIClient.java
+++ b/data-catalog-api/client/src/main/java/org/apache/airavata/datacatalog/api/client/DataCatalogAPIClient.java
@@ -7,11 +7,15 @@ import java.util.concurrent.TimeUnit;
import org.apache.airavata.datacatalog.api.DataCatalogAPIServiceGrpc;
import org.apache.airavata.datacatalog.api.DataCatalogAPIServiceGrpc.DataCatalogAPIServiceBlockingStub;
import org.apache.airavata.datacatalog.api.DataProduct;
+import org.apache.airavata.datacatalog.api.DataProductAddToMetadataSchemaRequest;
+import org.apache.airavata.datacatalog.api.DataProductAddToMetadataSchemaResponse;
import org.apache.airavata.datacatalog.api.DataProductCreateRequest;
import org.apache.airavata.datacatalog.api.DataProductCreateResponse;
import org.apache.airavata.datacatalog.api.DataProductDeleteRequest;
import org.apache.airavata.datacatalog.api.DataProductGetRequest;
import org.apache.airavata.datacatalog.api.DataProductGetResponse;
+import org.apache.airavata.datacatalog.api.DataProductRemoveFromMetadataSchemaRequest;
+import org.apache.airavata.datacatalog.api.DataProductRemoveFromMetadataSchemaResponse;
import org.apache.airavata.datacatalog.api.DataProductUpdateRequest;
import org.apache.airavata.datacatalog.api.DataProductUpdateResponse;
import org.apache.airavata.datacatalog.api.FieldValueType;
@@ -23,10 +27,14 @@ import org.apache.airavata.datacatalog.api.MetadataSchemaFieldCreateRequest;
import org.apache.airavata.datacatalog.api.MetadataSchemaFieldCreateResponse;
import org.apache.airavata.datacatalog.api.MetadataSchemaFieldListRequest;
import org.apache.airavata.datacatalog.api.MetadataSchemaFieldListResponse;
+import org.apache.airavata.datacatalog.api.MetadataSchemaGetRequest;
+import org.apache.airavata.datacatalog.api.MetadataSchemaGetResponse;
import io.grpc.Channel;
import io.grpc.ManagedChannel;
-import io.grpc.ManagedChannelBuilder;;
+import io.grpc.ManagedChannelBuilder;
+import io.grpc.Status;
+import io.grpc.StatusRuntimeException;;
public class DataCatalogAPIClient {
@@ -67,6 +75,19 @@ public class DataCatalogAPIClient {
return response.getMetadataSchema();
}
+ public MetadataSchema getMetadataSchema(String schemaName) {
+ MetadataSchemaGetRequest request = MetadataSchemaGetRequest.newBuilder().setSchemaName(schemaName).build();
+ try {
+ MetadataSchemaGetResponse response = blockingStub.getMetadataSchema(request);
+ return response.getMetadataSchema();
+ } catch (StatusRuntimeException e) {
+ if (e.getStatus() == Status.NOT_FOUND) {
+ return null;
+ }
+ throw e;
+ }
+ }
+
public MetadataSchemaField createMetadataSchemaField(MetadataSchemaField metadataSchemaField) {
MetadataSchemaFieldCreateRequest request = MetadataSchemaFieldCreateRequest.newBuilder()
.setMetadataSchemaField(metadataSchemaField).build();
@@ -81,6 +102,21 @@ public class DataCatalogAPIClient {
return response.getMetadataSchemaFieldsList();
}
+ public DataProduct addDataProductToMetadataSchema(String dataProductId, String schemaName) {
+ DataProductAddToMetadataSchemaRequest request = DataProductAddToMetadataSchemaRequest.newBuilder()
+ .setDataProductId(dataProductId).setSchemaName(schemaName).build();
+ DataProductAddToMetadataSchemaResponse response = blockingStub.addDataProductToMetadataSchema(request);
+ return response.getDataProduct();
+ }
+
+ public DataProduct removeDataProductFromMetadataSchema(String dataProductId, String schemaName) {
+ DataProductRemoveFromMetadataSchemaRequest request = DataProductRemoveFromMetadataSchemaRequest.newBuilder()
+ .setDataProductId(dataProductId).setSchemaName(schemaName).build();
+ DataProductRemoveFromMetadataSchemaResponse response = blockingStub
+ .removeDataProductFromMetadataSchema(request);
+ return response.getDataProduct();
+ }
+
public static void main(String[] args) throws InterruptedException {
String target = "localhost:6565";
@@ -115,11 +151,19 @@ public class DataCatalogAPIClient {
System.out.println(
MessageFormat.format("Deleted data product with id [{0}]", result2.getDataProductId()));
- MetadataSchema metadataSchema = MetadataSchema.newBuilder().setSchemaName("my_schema").build();
- metadataSchema = client.createMetadataSchema(metadataSchema);
-
- System.out.println(
- MessageFormat.format("Created metadata schema with name [{0}]", metadataSchema.getSchemaName()));
+ // First check if metadata schema exists
+ MetadataSchema metadataSchema = client.getMetadataSchema("my_schema");
+ if (metadataSchema == null) {
+ metadataSchema = MetadataSchema.newBuilder().setSchemaName("my_schema").build();
+ metadataSchema = client.createMetadataSchema(metadataSchema);
+ System.out.println(
+ MessageFormat.format("Created metadata schema with name [{0}]",
+ metadataSchema.getSchemaName()));
+ } else {
+ System.out.println(
+ MessageFormat.format("Found metadata schema with name [{0}]",
+ metadataSchema.getSchemaName()));
+ }
MetadataSchemaField field1 = MetadataSchemaField.newBuilder().setFieldName("field1")
.setJsonPath("$.field1").setValueType(FieldValueType.FLOAT)
@@ -142,6 +186,15 @@ public class DataCatalogAPIClient {
System.out.println(MessageFormat.format("-> field {0}", field.getFieldName()));
}
+ result = client.addDataProductToMetadataSchema(result.getDataProductId(), metadataSchema.getSchemaName());
+ System.out.println(MessageFormat.format("Added data product [{0}] to metadata schema [{1}]",
+ result.getDataProductId(), metadataSchema.getSchemaName()));
+
+ result = client.removeDataProductFromMetadataSchema(result.getDataProductId(),
+ metadataSchema.getSchemaName());
+ System.out.println(MessageFormat.format("Removed data product [{0}] from metadata schema [{1}]",
+ result.getDataProductId(), metadataSchema.getSchemaName()));
+
} finally {
channel.shutdownNow().awaitTermination(5, TimeUnit.SECONDS);
}
diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/mapper/DataProductMapper.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/mapper/DataProductMapper.java
index 8e1e155..b03fe1d 100644
--- a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/mapper/DataProductMapper.java
+++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/mapper/DataProductMapper.java
@@ -2,6 +2,7 @@ package org.apache.airavata.datacatalog.api.mapper;
import org.apache.airavata.datacatalog.api.DataProduct;
import org.apache.airavata.datacatalog.api.model.DataProductEntity;
+import org.apache.airavata.datacatalog.api.model.MetadataSchemaEntity;
import org.apache.airavata.datacatalog.api.repository.DataProductRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@@ -50,5 +51,10 @@ public class DataProductMapper {
if (dataProductEntity.getParentDataProductEntity() != null) {
dataProductBuilder.setParentDataProductId(dataProductEntity.getParentDataProductEntity().getExternalId());
}
+ if (dataProductEntity.getMetadataSchemas() != null) {
+ for (MetadataSchemaEntity metadataSchema : dataProductEntity.getMetadataSchemas()) {
+ dataProductBuilder.addSchemaName(metadataSchema.getSchemaName());
+ }
+ }
}
}
diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/model/DataProductEntity.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/model/DataProductEntity.java
index 65b24e7..68b615f 100644
--- a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/model/DataProductEntity.java
+++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/model/DataProductEntity.java
@@ -1,5 +1,7 @@
package org.apache.airavata.datacatalog.api.model;
+import java.util.Set;
+
import org.hibernate.annotations.Type;
import com.fasterxml.jackson.databind.JsonNode;
@@ -12,6 +14,8 @@ import jakarta.persistence.GeneratedValue;
import jakarta.persistence.GenerationType;
import jakarta.persistence.Id;
import jakarta.persistence.JoinColumn;
+import jakarta.persistence.JoinTable;
+import jakarta.persistence.ManyToMany;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.SequenceGenerator;
import jakarta.persistence.Table;
@@ -43,6 +47,10 @@ public class DataProductEntity {
@Column(name = "metadata", columnDefinition = "jsonb")
private JsonNode metadata;
+ @ManyToMany
+ @JoinTable(name = "data_product_metadata_schema", joinColumns = @JoinColumn(name = "data_product_id"), inverseJoinColumns = @JoinColumn(name = "metadata_schema_id"))
+ private Set<MetadataSchemaEntity> metadataSchemas;
+
// TODO: ManyToOne mapping to owner: UserEntity
public Long getDataProductId() {
return dataProductId;
@@ -84,6 +92,22 @@ public class DataProductEntity {
this.metadata = metadata;
}
+ public Set<MetadataSchemaEntity> getMetadataSchemas() {
+ return metadataSchemas;
+ }
+
+ public void setMetadataSchemas(Set<MetadataSchemaEntity> metadataSchemas) {
+ this.metadataSchemas = metadataSchemas;
+ }
+
+ public void addMetadataSchema(MetadataSchemaEntity metadataSchema) {
+ this.metadataSchemas.add(metadataSchema);
+ }
+
+ public void removeMetadataSchema(MetadataSchemaEntity metadataSchema) {
+ this.metadataSchemas.remove(metadataSchema);
+ }
+
@Override
public int hashCode() {
final int prime = 31;
diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/repository/DataProductRepository.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/repository/DataProductRepository.java
index acec036..426e4d4 100644
--- a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/repository/DataProductRepository.java
+++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/repository/DataProductRepository.java
@@ -4,6 +4,7 @@ import org.apache.airavata.datacatalog.api.model.DataProductEntity;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.transaction.annotation.Transactional;
+@Transactional(readOnly = true)
public interface DataProductRepository extends JpaRepository<DataProductEntity, Long> {
DataProductEntity findByExternalId(String externalId);
diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/service/DataCatalogAPIService.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/service/DataCatalogAPIService.java
index 3869ee5..7374def 100644
--- a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/service/DataCatalogAPIService.java
+++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/service/DataCatalogAPIService.java
@@ -28,6 +28,8 @@ import org.apache.airavata.datacatalog.api.MetadataSchemaFieldListRequest;
import org.apache.airavata.datacatalog.api.MetadataSchemaFieldListResponse;
import org.apache.airavata.datacatalog.api.MetadataSchemaFieldUpdateRequest;
import org.apache.airavata.datacatalog.api.MetadataSchemaFieldUpdateResponse;
+import org.apache.airavata.datacatalog.api.MetadataSchemaGetRequest;
+import org.apache.airavata.datacatalog.api.MetadataSchemaGetResponse;
import org.apache.airavata.datacatalog.api.mapper.DataProductMapper;
import org.apache.airavata.datacatalog.api.mapper.MetadataSchemaFieldMapper;
import org.apache.airavata.datacatalog.api.mapper.MetadataSchemaMapper;
@@ -41,10 +43,13 @@ import org.lognet.springboot.grpc.GRpcService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.transaction.annotation.Transactional;
+import io.grpc.Status;
import io.grpc.stub.StreamObserver;
@GRpcService
+@Transactional
public class DataCatalogAPIService extends DataCatalogAPIServiceGrpc.DataCatalogAPIServiceImplBase {
private static final Logger logger = LoggerFactory.getLogger(DataCatalogAPIService.class);
@@ -76,7 +81,7 @@ public class DataCatalogAPIService extends DataCatalogAPIServiceGrpc.DataCatalog
DataProductEntity dataProductEntity = new DataProductEntity();
dataProductEntity.setExternalId(UUID.randomUUID().toString());
dataProductMapper.mapModelToEntity(request.getDataProduct(), dataProductEntity);
- DataProductEntity savedDataProductEntity = dataProductRepository.save(dataProductEntity);
+ DataProductEntity savedDataProductEntity = dataProductRepository.saveAndFlush(dataProductEntity);
// TODO: SharingManager.grantPermissionToUser(userInfo, dataProduct,
// Permission.OWNER)
@@ -131,8 +136,33 @@ public class DataCatalogAPIService extends DataCatalogAPIServiceGrpc.DataCatalog
@Override
public void addDataProductToMetadataSchema(DataProductAddToMetadataSchemaRequest request,
StreamObserver<DataProductAddToMetadataSchemaResponse> responseObserver) {
- // TODO Auto-generated method stub
- super.addDataProductToMetadataSchema(request, responseObserver);
+ String dataProductId = request.getDataProductId();
+ // TODO: handle data product not found
+ DataProductEntity dataProduct = dataProductRepository.findByExternalId(dataProductId);
+ String schemaName = request.getSchemaName();
+ // TODO: handle metadata schema not found
+ MetadataSchemaEntity metadataSchemaEntity = metadataSchemaRepository.findBySchemaName(schemaName);
+ dataProduct.addMetadataSchema(metadataSchemaEntity);
+
+ DataProductAddToMetadataSchemaResponse.Builder responseBuilder = DataProductAddToMetadataSchemaResponse
+ .newBuilder();
+ dataProductMapper.mapEntityToModel(dataProduct, responseBuilder.getDataProductBuilder());
+ responseObserver.onNext(responseBuilder.build());
+ responseObserver.onCompleted();
+ }
+
+ @Override
+ public void getMetadataSchema(MetadataSchemaGetRequest request,
+ StreamObserver<MetadataSchemaGetResponse> responseObserver) {
+ MetadataSchemaEntity metadataSchemaEntity = metadataSchemaRepository.findBySchemaName(request.getSchemaName());
+ if (metadataSchemaEntity == null) {
+ responseObserver.onError(Status.NOT_FOUND.asException());
+ responseObserver.onCompleted();
+ }
+ MetadataSchemaGetResponse.Builder responseBuilder = MetadataSchemaGetResponse.newBuilder();
+ metadataSchemaMapper.mapEntityToModel(metadataSchemaEntity, responseBuilder.getMetadataSchemaBuilder());
+ responseObserver.onNext(responseBuilder.build());
+ responseObserver.onCompleted();
}
@Override
@@ -193,15 +223,26 @@ public class DataCatalogAPIService extends DataCatalogAPIServiceGrpc.DataCatalog
metadataSchemaFieldMapper.mapEntityToModel(metadataSchemaFieldEntity, builder);
}
responseObserver.onNext(responseBuilder.build());
- responseObserver.onError(null);
responseObserver.onCompleted();
}
@Override
public void removeDataProductFromMetadataSchema(DataProductRemoveFromMetadataSchemaRequest request,
StreamObserver<DataProductRemoveFromMetadataSchemaResponse> responseObserver) {
- // TODO Auto-generated method stub
- super.removeDataProductFromMetadataSchema(request, responseObserver);
+
+ String dataProductId = request.getDataProductId();
+ // TODO: handle data product not found
+ DataProductEntity dataProduct = dataProductRepository.findByExternalId(dataProductId);
+ String schemaName = request.getSchemaName();
+ // TODO: handle metadata schema not found
+ MetadataSchemaEntity metadataSchemaEntity = metadataSchemaRepository.findBySchemaName(schemaName);
+ dataProduct.removeMetadataSchema(metadataSchemaEntity);
+
+ DataProductRemoveFromMetadataSchemaResponse.Builder responseBuilder = DataProductRemoveFromMetadataSchemaResponse
+ .newBuilder();
+ dataProductMapper.mapEntityToModel(dataProduct, responseBuilder.getDataProductBuilder());
+ responseObserver.onNext(responseBuilder.build());
+ responseObserver.onCompleted();
}
@Override
diff --git a/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto b/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto
index b1d40dd..cb03016 100644
--- a/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto
+++ b/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto
@@ -62,6 +62,7 @@ message DataProduct {
optional string parent_data_product_id = 2;
string name = 3;
optional string metadata = 4;
+ repeated string schema_name = 5;
}
enum FieldValueType {
@@ -105,6 +106,13 @@ message DataProductDeleteRequest {
message DataProductDeleteResponse {
}
+message MetadataSchemaGetRequest {
+ UserInfo user_info = 1;
+ string schema_name = 2;
+}
+message MetadataSchemaGetResponse {
+ MetadataSchema metadata_schema = 1;
+}
message MetadataSchemaCreateRequest {
UserInfo user_info = 1;
MetadataSchema metadata_schema = 2;
@@ -168,6 +176,7 @@ service DataCatalogAPIService {
rpc updateDataProduct(DataProductUpdateRequest) returns (DataProductUpdateResponse){}
rpc getDataProduct(DataProductGetRequest) returns (DataProductGetResponse){}
rpc deleteDataProduct(DataProductDeleteRequest) returns (DataProductDeleteResponse){}
+ rpc getMetadataSchema(MetadataSchemaGetRequest) returns (MetadataSchemaGetResponse){}
rpc createMetadataSchema(MetadataSchemaCreateRequest) returns (MetadataSchemaCreateResponse){}
rpc deleteMetadataSchema(MetadataSchemaCreateRequest) returns (MetadataSchemaCreateResponse){}
rpc createMetadataSchemaField(MetadataSchemaFieldCreateRequest) returns (MetadataSchemaFieldCreateResponse){}
[airavata-data-catalog] 03/05: moved test package
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch machristie/issue7
in repository https://gitbox.apache.org/repos/asf/airavata-data-catalog.git
commit 011dff88095c38a309db9d43ed4266dc2381b204
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Wed Feb 15 15:31:38 2023 -0500
moved test package
---
.../api}/DataCatalogApiServerApplicationTests.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/data-catalog-api/server/src/test/java/org/apache/airavata/datacatalogapiserver/DataCatalogApiServerApplicationTests.java b/data-catalog-api/server/src/test/java/org/apache/airavata/datacatalog/api/DataCatalogApiServerApplicationTests.java
similarity index 79%
rename from data-catalog-api/server/src/test/java/org/apache/airavata/datacatalogapiserver/DataCatalogApiServerApplicationTests.java
rename to data-catalog-api/server/src/test/java/org/apache/airavata/datacatalog/api/DataCatalogApiServerApplicationTests.java
index f7ed907..00bd885 100644
--- a/data-catalog-api/server/src/test/java/org/apache/airavata/datacatalogapiserver/DataCatalogApiServerApplicationTests.java
+++ b/data-catalog-api/server/src/test/java/org/apache/airavata/datacatalog/api/DataCatalogApiServerApplicationTests.java
@@ -1,4 +1,4 @@
-package org.apache.airavata.datacatalogapiserver;
+package org.apache.airavata.datacatalog.api;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
[airavata-data-catalog] 04/05: Support creating data product for known metadata schemas
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch machristie/issue7
in repository https://gitbox.apache.org/repos/asf/airavata-data-catalog.git
commit 9d7f93b3229ac3665c94fef241fc2ca5bf588624
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Wed Feb 15 15:45:45 2023 -0500
Support creating data product for known metadata schemas
---
.../datacatalog/api/client/DataCatalogAPIClient.java | 11 +++++++++++
.../datacatalog/api/mapper/DataProductMapper.java | 16 +++++++++++++++-
.../datacatalog/api/model/DataProductEntity.java | 3 ++-
.../stubs/src/main/proto/DataCatalogAPI.proto | 2 +-
4 files changed, 29 insertions(+), 3 deletions(-)
diff --git a/data-catalog-api/client/src/main/java/org/apache/airavata/datacatalog/api/client/DataCatalogAPIClient.java b/data-catalog-api/client/src/main/java/org/apache/airavata/datacatalog/api/client/DataCatalogAPIClient.java
index 3d59fdb..25495fd 100644
--- a/data-catalog-api/client/src/main/java/org/apache/airavata/datacatalog/api/client/DataCatalogAPIClient.java
+++ b/data-catalog-api/client/src/main/java/org/apache/airavata/datacatalog/api/client/DataCatalogAPIClient.java
@@ -195,6 +195,17 @@ public class DataCatalogAPIClient {
System.out.println(MessageFormat.format("Removed data product [{0}] from metadata schema [{1}]",
result.getDataProductId(), metadataSchema.getSchemaName()));
+ // Create data product that belongs to my_schema schema
+ DataProduct dataProduct3 = DataProduct.newBuilder()
+ .setName("testing 3")
+ .setMetadata("{\"foo\": \"bar\"}")
+ .addMetadataSchemas("my_schema")
+ .build();
+ DataProduct result3 = client.createDataProduct(dataProduct3);
+ System.out.println(
+ MessageFormat.format("Created third data product [{0}], supporting schemas [{1}]",
+ result3.getDataProductId(), result3.getMetadataSchemasList()));
+
} finally {
channel.shutdownNow().awaitTermination(5, TimeUnit.SECONDS);
}
diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/mapper/DataProductMapper.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/mapper/DataProductMapper.java
index b03fe1d..b6817ea 100644
--- a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/mapper/DataProductMapper.java
+++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/mapper/DataProductMapper.java
@@ -4,6 +4,7 @@ import org.apache.airavata.datacatalog.api.DataProduct;
import org.apache.airavata.datacatalog.api.model.DataProductEntity;
import org.apache.airavata.datacatalog.api.model.MetadataSchemaEntity;
import org.apache.airavata.datacatalog.api.repository.DataProductRepository;
+import org.apache.airavata.datacatalog.api.repository.MetadataSchemaRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@@ -22,6 +23,9 @@ public class DataProductMapper {
@Autowired
DataProductRepository dataProductRepository;
+ @Autowired
+ MetadataSchemaRepository metadataSchemaRepository;
+
public void mapModelToEntity(DataProduct dataProduct, DataProductEntity dataProductEntity) {
dataProductEntity.setName(dataProduct.getName());
@@ -41,6 +45,16 @@ public class DataProductMapper {
throw new RuntimeException(e);
}
}
+
+ // Synchronize the list of metadata schemas
+ if (dataProductEntity.getMetadataSchemas() != null) {
+ dataProductEntity.getMetadataSchemas().clear();
+ }
+ for (String metadataSchemaName : dataProduct.getMetadataSchemasList()) {
+ // TODO: handle metadata schema not found
+ MetadataSchemaEntity metadataSchema = metadataSchemaRepository.findBySchemaName(metadataSchemaName);
+ dataProductEntity.addMetadataSchema(metadataSchema);
+ }
}
public void mapEntityToModel(DataProductEntity dataProductEntity, DataProduct.Builder dataProductBuilder) {
@@ -53,7 +67,7 @@ public class DataProductMapper {
}
if (dataProductEntity.getMetadataSchemas() != null) {
for (MetadataSchemaEntity metadataSchema : dataProductEntity.getMetadataSchemas()) {
- dataProductBuilder.addSchemaName(metadataSchema.getSchemaName());
+ dataProductBuilder.addMetadataSchemas(metadataSchema.getSchemaName());
}
}
}
diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/model/DataProductEntity.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/model/DataProductEntity.java
index 68b615f..5e3802f 100644
--- a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/model/DataProductEntity.java
+++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/model/DataProductEntity.java
@@ -1,5 +1,6 @@
package org.apache.airavata.datacatalog.api.model;
+import java.util.HashSet;
import java.util.Set;
import org.hibernate.annotations.Type;
@@ -49,7 +50,7 @@ public class DataProductEntity {
@ManyToMany
@JoinTable(name = "data_product_metadata_schema", joinColumns = @JoinColumn(name = "data_product_id"), inverseJoinColumns = @JoinColumn(name = "metadata_schema_id"))
- private Set<MetadataSchemaEntity> metadataSchemas;
+ private Set<MetadataSchemaEntity> metadataSchemas = new HashSet<>();
// TODO: ManyToOne mapping to owner: UserEntity
public Long getDataProductId() {
diff --git a/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto b/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto
index cb03016..7d91b8b 100644
--- a/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto
+++ b/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto
@@ -62,7 +62,7 @@ message DataProduct {
optional string parent_data_product_id = 2;
string name = 3;
optional string metadata = 4;
- repeated string schema_name = 5;
+ repeated string metadata_schemas = 5;
}
enum FieldValueType {
[airavata-data-catalog] 01/05: Initial work at implementing metadata schema management APIs
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch machristie/issue7
in repository https://gitbox.apache.org/repos/asf/airavata-data-catalog.git
commit 131ca0eabdfebb7573812614af75c6d4d12ff835
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Mon Feb 13 17:01:28 2023 -0500
Initial work at implementing metadata schema management APIs
---
.../api/client/DataCatalogAPIClient.java | 62 ++++++++++-
.../api/mapper/MetadataSchemaFieldMapper.java | 36 +++++++
.../api/mapper/MetadataSchemaMapper.java | 19 ++++
.../api/model/MetadataSchemaEntity.java | 73 +++++++++++++
.../api/model/MetadataSchemaFieldEntity.java | 108 +++++++++++++++++++
.../repository/MetadataSchemaFieldRepository.java | 11 ++
.../api/repository/MetadataSchemaRepository.java | 9 ++
.../api/service/DataCatalogAPIService.java | 119 ++++++++++++++++++++-
.../stubs/src/main/proto/DataCatalogAPI.proto | 83 ++++++++++++++
9 files changed, 515 insertions(+), 5 deletions(-)
diff --git a/data-catalog-api/client/src/main/java/org/apache/airavata/datacatalog/api/client/DataCatalogAPIClient.java b/data-catalog-api/client/src/main/java/org/apache/airavata/datacatalog/api/client/DataCatalogAPIClient.java
index 8ae7d07..954f994 100644
--- a/data-catalog-api/client/src/main/java/org/apache/airavata/datacatalog/api/client/DataCatalogAPIClient.java
+++ b/data-catalog-api/client/src/main/java/org/apache/airavata/datacatalog/api/client/DataCatalogAPIClient.java
@@ -1,6 +1,7 @@
package org.apache.airavata.datacatalog.api.client;
import java.text.MessageFormat;
+import java.util.List;
import java.util.concurrent.TimeUnit;
import org.apache.airavata.datacatalog.api.DataCatalogAPIServiceGrpc;
@@ -13,8 +14,15 @@ import org.apache.airavata.datacatalog.api.DataProductGetRequest;
import org.apache.airavata.datacatalog.api.DataProductGetResponse;
import org.apache.airavata.datacatalog.api.DataProductUpdateRequest;
import org.apache.airavata.datacatalog.api.DataProductUpdateResponse;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.airavata.datacatalog.api.FieldValueType;
+import org.apache.airavata.datacatalog.api.MetadataSchema;
+import org.apache.airavata.datacatalog.api.MetadataSchemaCreateRequest;
+import org.apache.airavata.datacatalog.api.MetadataSchemaCreateResponse;
+import org.apache.airavata.datacatalog.api.MetadataSchemaField;
+import org.apache.airavata.datacatalog.api.MetadataSchemaFieldCreateRequest;
+import org.apache.airavata.datacatalog.api.MetadataSchemaFieldCreateResponse;
+import org.apache.airavata.datacatalog.api.MetadataSchemaFieldListRequest;
+import org.apache.airavata.datacatalog.api.MetadataSchemaFieldListResponse;
import io.grpc.Channel;
import io.grpc.ManagedChannel;
@@ -22,8 +30,6 @@ import io.grpc.ManagedChannelBuilder;;
public class DataCatalogAPIClient {
- private static final Logger logger = LoggerFactory.getLogger(DataCatalogAPIClient.class);
-
private final DataCatalogAPIServiceBlockingStub blockingStub;
public DataCatalogAPIClient(Channel channel) {
@@ -54,6 +60,27 @@ public class DataCatalogAPIClient {
blockingStub.deleteDataProduct(request);
}
+ public MetadataSchema createMetadataSchema(MetadataSchema metadataSchema) {
+ MetadataSchemaCreateRequest request = MetadataSchemaCreateRequest.newBuilder().setMetadataSchema(metadataSchema)
+ .build();
+ MetadataSchemaCreateResponse response = blockingStub.createMetadataSchema(request);
+ return response.getMetadataSchema();
+ }
+
+ public MetadataSchemaField createMetadataSchemaField(MetadataSchemaField metadataSchemaField) {
+ MetadataSchemaFieldCreateRequest request = MetadataSchemaFieldCreateRequest.newBuilder()
+ .setMetadataSchemaField(metadataSchemaField).build();
+ MetadataSchemaFieldCreateResponse response = blockingStub.createMetadataSchemaField(request);
+ return response.getMetadataSchemaField();
+ }
+
+ public List<MetadataSchemaField> getMetadataSchemaFields(String schemaName) {
+ MetadataSchemaFieldListRequest request = MetadataSchemaFieldListRequest.newBuilder().setSchemaName(schemaName)
+ .build();
+ MetadataSchemaFieldListResponse response = blockingStub.getMetadataSchemaFields(request);
+ return response.getMetadataSchemaFieldsList();
+ }
+
public static void main(String[] args) throws InterruptedException {
String target = "localhost:6565";
@@ -88,6 +115,33 @@ public class DataCatalogAPIClient {
System.out.println(
MessageFormat.format("Deleted data product with id [{0}]", result2.getDataProductId()));
+ MetadataSchema metadataSchema = MetadataSchema.newBuilder().setSchemaName("my_schema").build();
+ metadataSchema = client.createMetadataSchema(metadataSchema);
+
+ System.out.println(
+ MessageFormat.format("Created metadata schema with name [{0}]", metadataSchema.getSchemaName()));
+
+ MetadataSchemaField field1 = MetadataSchemaField.newBuilder().setFieldName("field1")
+ .setJsonPath("$.field1").setValueType(FieldValueType.FLOAT)
+ .setSchemaName(metadataSchema.getSchemaName()).build();
+ field1 = client.createMetadataSchemaField(field1);
+ System.out.println(MessageFormat.format("Created metadata schema field [{0}] in schema [{1}]",
+ field1.getFieldName(), field1.getSchemaName()));
+
+ MetadataSchemaField field2 = MetadataSchemaField.newBuilder().setFieldName("field2")
+ .setJsonPath("$.field2").setValueType(FieldValueType.FLOAT)
+ .setSchemaName(metadataSchema.getSchemaName()).build();
+ field2 = client.createMetadataSchemaField(field2);
+ System.out.println(MessageFormat.format("Created metadata schema field [{0}] in schema [{1}]",
+ field2.getFieldName(), field2.getSchemaName()));
+
+ List<MetadataSchemaField> fields = client.getMetadataSchemaFields(metadataSchema.getSchemaName());
+ System.out.println(MessageFormat.format("Found {0} fields for schema {1}", fields.size(),
+ metadataSchema.getSchemaName()));
+ for (MetadataSchemaField field : fields) {
+ System.out.println(MessageFormat.format("-> field {0}", field.getFieldName()));
+ }
+
} finally {
channel.shutdownNow().awaitTermination(5, TimeUnit.SECONDS);
}
diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/mapper/MetadataSchemaFieldMapper.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/mapper/MetadataSchemaFieldMapper.java
new file mode 100644
index 0000000..2a96ec4
--- /dev/null
+++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/mapper/MetadataSchemaFieldMapper.java
@@ -0,0 +1,36 @@
+package org.apache.airavata.datacatalog.api.mapper;
+
+import org.apache.airavata.datacatalog.api.MetadataSchemaField;
+import org.apache.airavata.datacatalog.api.model.MetadataSchemaEntity;
+import org.apache.airavata.datacatalog.api.model.MetadataSchemaFieldEntity;
+import org.apache.airavata.datacatalog.api.repository.MetadataSchemaRepository;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class MetadataSchemaFieldMapper {
+
+ @Autowired
+ MetadataSchemaRepository metadataSchemaRepository;
+
+ public void mapModelToEntity(MetadataSchemaField metadataSchemaField,
+ MetadataSchemaFieldEntity metadataSchemaFieldEntity) {
+
+ metadataSchemaFieldEntity.setFieldName(metadataSchemaField.getFieldName());
+ metadataSchemaFieldEntity.setFieldValueType(metadataSchemaField.getValueType());
+ metadataSchemaFieldEntity.setJsonPath(metadataSchemaField.getJsonPath());
+
+ MetadataSchemaEntity metadataSchemaEntity = metadataSchemaRepository
+ .findBySchemaName(metadataSchemaField.getSchemaName());
+ metadataSchemaFieldEntity.setMetadataSchema(metadataSchemaEntity);
+ }
+
+ public void mapEntityToModel(MetadataSchemaFieldEntity metadataSchemaFieldEntity,
+ MetadataSchemaField.Builder metadataSchemaFieldBuilder) {
+
+ metadataSchemaFieldBuilder.setFieldName(metadataSchemaFieldEntity.getFieldName());
+ metadataSchemaFieldBuilder.setJsonPath(metadataSchemaFieldEntity.getJsonPath());
+ metadataSchemaFieldBuilder.setValueType(metadataSchemaFieldEntity.getFieldValueType());
+ metadataSchemaFieldBuilder.setSchemaName(metadataSchemaFieldEntity.getMetadataSchema().getSchemaName());
+ }
+}
diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/mapper/MetadataSchemaMapper.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/mapper/MetadataSchemaMapper.java
new file mode 100644
index 0000000..e7c6fcd
--- /dev/null
+++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/mapper/MetadataSchemaMapper.java
@@ -0,0 +1,19 @@
+package org.apache.airavata.datacatalog.api.mapper;
+
+import org.apache.airavata.datacatalog.api.MetadataSchema;
+import org.apache.airavata.datacatalog.api.model.MetadataSchemaEntity;
+import org.springframework.stereotype.Component;
+
+@Component
+public class MetadataSchemaMapper {
+
+ public void mapModelToEntity(MetadataSchema metadataSchema, MetadataSchemaEntity metadataSchemaEntity) {
+
+ metadataSchemaEntity.setSchemaName(metadataSchema.getSchemaName());
+ }
+
+ public void mapEntityToModel(MetadataSchemaEntity metadataSchemaEntity,
+ MetadataSchema.Builder metadataSchemaBuilder) {
+ metadataSchemaBuilder.setSchemaName(metadataSchemaEntity.getSchemaName());
+ }
+}
diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/model/MetadataSchemaEntity.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/model/MetadataSchemaEntity.java
new file mode 100644
index 0000000..8c1f6cd
--- /dev/null
+++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/model/MetadataSchemaEntity.java
@@ -0,0 +1,73 @@
+package org.apache.airavata.datacatalog.api.model;
+
+import java.util.Set;
+
+import jakarta.persistence.Basic;
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.GeneratedValue;
+import jakarta.persistence.GenerationType;
+import jakarta.persistence.Id;
+import jakarta.persistence.OneToMany;
+import jakarta.persistence.SequenceGenerator;
+import jakarta.persistence.Table;
+
+@Entity
+// TODO: unique constraint on schema_name, tenant
+@Table(name = "metadata_schema")
+public class MetadataSchemaEntity {
+
+ @Id
+ @SequenceGenerator(name = "metadata_schema_metadata_schema_id_seq", sequenceName = "metadata_schema_metadata_schema_id_seq", allocationSize = 1)
+ @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "metadata_schema_metadata_schema_id_seq")
+ @Column(name = "metadata_schema_id")
+ private Long metadataSchemaId;
+
+ @Basic
+ @Column(name = "schema_name", nullable = false)
+ private String schemaName;
+
+ @OneToMany(mappedBy = "metadataSchema")
+ private Set<MetadataSchemaFieldEntity> metadataSchemaFields;
+
+ public Long getMetadataSchemaId() {
+ return metadataSchemaId;
+ }
+
+ public void setMetadataSchemaId(Long metadataSchemaId) {
+ this.metadataSchemaId = metadataSchemaId;
+ }
+
+ public String getSchemaName() {
+ return schemaName;
+ }
+
+ public void setSchemaName(String schemaName) {
+ this.schemaName = schemaName;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((metadataSchemaId == null) ? 0 : metadataSchemaId.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ MetadataSchemaEntity other = (MetadataSchemaEntity) obj;
+ if (metadataSchemaId == null) {
+ if (other.metadataSchemaId != null)
+ return false;
+ } else if (!metadataSchemaId.equals(other.metadataSchemaId))
+ return false;
+ return true;
+ }
+}
diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/model/MetadataSchemaFieldEntity.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/model/MetadataSchemaFieldEntity.java
new file mode 100644
index 0000000..8f6a83a
--- /dev/null
+++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/model/MetadataSchemaFieldEntity.java
@@ -0,0 +1,108 @@
+package org.apache.airavata.datacatalog.api.model;
+
+import org.apache.airavata.datacatalog.api.FieldValueType;
+
+import jakarta.persistence.Basic;
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.EnumType;
+import jakarta.persistence.Enumerated;
+import jakarta.persistence.GeneratedValue;
+import jakarta.persistence.GenerationType;
+import jakarta.persistence.Id;
+import jakarta.persistence.JoinColumn;
+import jakarta.persistence.ManyToOne;
+import jakarta.persistence.SequenceGenerator;
+import jakarta.persistence.Table;
+
+@Entity
+@Table(name = "metadata_schema_field")
+public class MetadataSchemaFieldEntity {
+
+ @Id
+ @SequenceGenerator(name = "metadata_schema_field_metadata_schema_field_id", sequenceName = "metadata_schema_field_metadata_schema_field_id", allocationSize = 1)
+ @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = "metadata_schema_field_metadata_schema_field_id")
+ @Column(name = "metadata_schema_field_id")
+ private Long metadataSchemaFieldId;
+
+ @Basic
+ @Column(name = "field_name", nullable = false)
+ private String fieldName;
+
+ @Basic
+ @Column(name = "json_path", nullable = false)
+ private String jsonPath;
+
+ @Enumerated(EnumType.STRING)
+ private FieldValueType fieldValueType;
+
+ @ManyToOne(optional = false)
+ @JoinColumn(name = "metadata_schema_id", nullable = false, updatable = false)
+ private MetadataSchemaEntity metadataSchema;
+
+ public Long getMetadataSchemaFieldId() {
+ return metadataSchemaFieldId;
+ }
+
+ public void setMetadataSchemaFieldId(Long metadataSchemaFieldId) {
+ this.metadataSchemaFieldId = metadataSchemaFieldId;
+ }
+
+ public String getFieldName() {
+ return fieldName;
+ }
+
+ public void setFieldName(String fieldName) {
+ this.fieldName = fieldName;
+ }
+
+ public String getJsonPath() {
+ return jsonPath;
+ }
+
+ public void setJsonPath(String jsonPath) {
+ this.jsonPath = jsonPath;
+ }
+
+ public FieldValueType getFieldValueType() {
+ return fieldValueType;
+ }
+
+ public void setFieldValueType(FieldValueType fieldValueType) {
+ this.fieldValueType = fieldValueType;
+ }
+
+ public MetadataSchemaEntity getMetadataSchema() {
+ return metadataSchema;
+ }
+
+ public void setMetadataSchema(MetadataSchemaEntity metadataSchema) {
+ this.metadataSchema = metadataSchema;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((metadataSchemaFieldId == null) ? 0 : metadataSchemaFieldId.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ MetadataSchemaFieldEntity other = (MetadataSchemaFieldEntity) obj;
+ if (metadataSchemaFieldId == null) {
+ if (other.metadataSchemaFieldId != null)
+ return false;
+ } else if (!metadataSchemaFieldId.equals(other.metadataSchemaFieldId))
+ return false;
+ return true;
+ }
+
+}
diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/repository/MetadataSchemaFieldRepository.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/repository/MetadataSchemaFieldRepository.java
new file mode 100644
index 0000000..e237dac
--- /dev/null
+++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/repository/MetadataSchemaFieldRepository.java
@@ -0,0 +1,11 @@
+package org.apache.airavata.datacatalog.api.repository;
+
+import java.util.List;
+
+import org.apache.airavata.datacatalog.api.model.MetadataSchemaFieldEntity;
+import org.springframework.data.jpa.repository.JpaRepository;
+
+public interface MetadataSchemaFieldRepository extends JpaRepository<MetadataSchemaFieldEntity, Long> {
+
+ List<MetadataSchemaFieldEntity> findByMetadataSchema_SchemaName(String schemaName);
+}
diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/repository/MetadataSchemaRepository.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/repository/MetadataSchemaRepository.java
new file mode 100644
index 0000000..28d0518
--- /dev/null
+++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/repository/MetadataSchemaRepository.java
@@ -0,0 +1,9 @@
+package org.apache.airavata.datacatalog.api.repository;
+
+import org.apache.airavata.datacatalog.api.model.MetadataSchemaEntity;
+import org.springframework.data.jpa.repository.JpaRepository;
+
+public interface MetadataSchemaRepository extends JpaRepository<MetadataSchemaEntity, Long> {
+
+ MetadataSchemaEntity findBySchemaName(String schemaName);
+}
diff --git a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/service/DataCatalogAPIService.java b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/service/DataCatalogAPIService.java
index 65f92b7..3869ee5 100644
--- a/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/service/DataCatalogAPIService.java
+++ b/data-catalog-api/server/src/main/java/org/apache/airavata/datacatalog/api/service/DataCatalogAPIService.java
@@ -1,20 +1,42 @@
package org.apache.airavata.datacatalog.api.service;
+import java.util.List;
import java.util.UUID;
import org.apache.airavata.datacatalog.api.DataCatalogAPIServiceGrpc;
import org.apache.airavata.datacatalog.api.DataProduct;
+import org.apache.airavata.datacatalog.api.DataProductAddToMetadataSchemaRequest;
+import org.apache.airavata.datacatalog.api.DataProductAddToMetadataSchemaResponse;
import org.apache.airavata.datacatalog.api.DataProductCreateRequest;
import org.apache.airavata.datacatalog.api.DataProductCreateResponse;
import org.apache.airavata.datacatalog.api.DataProductDeleteRequest;
import org.apache.airavata.datacatalog.api.DataProductDeleteResponse;
import org.apache.airavata.datacatalog.api.DataProductGetRequest;
import org.apache.airavata.datacatalog.api.DataProductGetResponse;
+import org.apache.airavata.datacatalog.api.DataProductRemoveFromMetadataSchemaRequest;
+import org.apache.airavata.datacatalog.api.DataProductRemoveFromMetadataSchemaResponse;
import org.apache.airavata.datacatalog.api.DataProductUpdateRequest;
import org.apache.airavata.datacatalog.api.DataProductUpdateResponse;
+import org.apache.airavata.datacatalog.api.MetadataSchemaCreateRequest;
+import org.apache.airavata.datacatalog.api.MetadataSchemaCreateResponse;
+import org.apache.airavata.datacatalog.api.MetadataSchemaField;
+import org.apache.airavata.datacatalog.api.MetadataSchemaFieldCreateRequest;
+import org.apache.airavata.datacatalog.api.MetadataSchemaFieldCreateResponse;
+import org.apache.airavata.datacatalog.api.MetadataSchemaFieldDeleteRequest;
+import org.apache.airavata.datacatalog.api.MetadataSchemaFieldDeleteResponse;
+import org.apache.airavata.datacatalog.api.MetadataSchemaFieldListRequest;
+import org.apache.airavata.datacatalog.api.MetadataSchemaFieldListResponse;
+import org.apache.airavata.datacatalog.api.MetadataSchemaFieldUpdateRequest;
+import org.apache.airavata.datacatalog.api.MetadataSchemaFieldUpdateResponse;
import org.apache.airavata.datacatalog.api.mapper.DataProductMapper;
+import org.apache.airavata.datacatalog.api.mapper.MetadataSchemaFieldMapper;
+import org.apache.airavata.datacatalog.api.mapper.MetadataSchemaMapper;
import org.apache.airavata.datacatalog.api.model.DataProductEntity;
+import org.apache.airavata.datacatalog.api.model.MetadataSchemaEntity;
+import org.apache.airavata.datacatalog.api.model.MetadataSchemaFieldEntity;
import org.apache.airavata.datacatalog.api.repository.DataProductRepository;
+import org.apache.airavata.datacatalog.api.repository.MetadataSchemaFieldRepository;
+import org.apache.airavata.datacatalog.api.repository.MetadataSchemaRepository;
import org.lognet.springboot.grpc.GRpcService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -31,7 +53,19 @@ public class DataCatalogAPIService extends DataCatalogAPIServiceGrpc.DataCatalog
DataProductRepository dataProductRepository;
@Autowired
- DataProductMapper dataProductMapper = new DataProductMapper();
+ MetadataSchemaRepository metadataSchemaRepository;
+
+ @Autowired
+ MetadataSchemaFieldRepository metadataSchemaFieldRepository;
+
+ @Autowired
+ DataProductMapper dataProductMapper;
+
+ @Autowired
+ MetadataSchemaMapper metadataSchemaMapper;
+
+ @Autowired
+ MetadataSchemaFieldMapper metadataSchemaFieldMapper;
@Override
public void createDataProduct(DataProductCreateRequest request,
@@ -93,4 +127,87 @@ public class DataCatalogAPIService extends DataCatalogAPIServiceGrpc.DataCatalog
responseObserver.onNext(responseBuilder.build());
responseObserver.onCompleted();
}
+
+ @Override
+ public void addDataProductToMetadataSchema(DataProductAddToMetadataSchemaRequest request,
+ StreamObserver<DataProductAddToMetadataSchemaResponse> responseObserver) {
+ // TODO Auto-generated method stub
+ super.addDataProductToMetadataSchema(request, responseObserver);
+ }
+
+ @Override
+ public void createMetadataSchema(MetadataSchemaCreateRequest request,
+ StreamObserver<MetadataSchemaCreateResponse> responseObserver) {
+
+ MetadataSchemaEntity metadataSchemaEntity = new MetadataSchemaEntity();
+ metadataSchemaMapper.mapModelToEntity(request.getMetadataSchema(), metadataSchemaEntity);
+ MetadataSchemaEntity savedMetadataSchemaEntity = metadataSchemaRepository.save(metadataSchemaEntity);
+
+ MetadataSchemaCreateResponse.Builder responseBuilder = MetadataSchemaCreateResponse.newBuilder();
+ metadataSchemaMapper.mapEntityToModel(savedMetadataSchemaEntity, responseBuilder.getMetadataSchemaBuilder());
+ responseObserver.onNext(responseBuilder.build());
+ responseObserver.onCompleted();
+ }
+
+ @Override
+ public void createMetadataSchemaField(MetadataSchemaFieldCreateRequest request,
+ StreamObserver<MetadataSchemaFieldCreateResponse> responseObserver) {
+
+ MetadataSchemaFieldEntity metadataSchemaFieldEntity = new MetadataSchemaFieldEntity();
+ metadataSchemaFieldMapper.mapModelToEntity(request.getMetadataSchemaField(), metadataSchemaFieldEntity);
+ MetadataSchemaFieldEntity savedMetadataSchemaFieldEntity = metadataSchemaFieldRepository
+ .save(metadataSchemaFieldEntity);
+
+ MetadataSchemaFieldCreateResponse.Builder responseBuilder = MetadataSchemaFieldCreateResponse.newBuilder();
+ metadataSchemaFieldMapper.mapEntityToModel(savedMetadataSchemaFieldEntity,
+ responseBuilder.getMetadataSchemaFieldBuilder());
+ responseObserver.onNext(responseBuilder.build());
+ responseObserver.onCompleted();
+ }
+
+ @Override
+ public void deleteMetadataSchema(MetadataSchemaCreateRequest request,
+ StreamObserver<MetadataSchemaCreateResponse> responseObserver) {
+ // TODO Auto-generated method stub
+ super.deleteMetadataSchema(request, responseObserver);
+ }
+
+ @Override
+ public void deleteMetadataSchemaField(MetadataSchemaFieldDeleteRequest request,
+ StreamObserver<MetadataSchemaFieldDeleteResponse> responseObserver) {
+ // TODO Auto-generated method stub
+ super.deleteMetadataSchemaField(request, responseObserver);
+ }
+
+ @Override
+ public void getMetadataSchemaFields(MetadataSchemaFieldListRequest request,
+ StreamObserver<MetadataSchemaFieldListResponse> responseObserver) {
+
+ // TODO: handle case where schema doesn't exist
+ List<MetadataSchemaFieldEntity> metadataSchemaFieldEntities = metadataSchemaFieldRepository
+ .findByMetadataSchema_SchemaName(request.getSchemaName());
+
+ MetadataSchemaFieldListResponse.Builder responseBuilder = MetadataSchemaFieldListResponse.newBuilder();
+ for (MetadataSchemaFieldEntity metadataSchemaFieldEntity : metadataSchemaFieldEntities) {
+ MetadataSchemaField.Builder builder = responseBuilder.addMetadataSchemaFieldsBuilder();
+ metadataSchemaFieldMapper.mapEntityToModel(metadataSchemaFieldEntity, builder);
+ }
+ responseObserver.onNext(responseBuilder.build());
+ responseObserver.onError(null);
+ responseObserver.onCompleted();
+ }
+
+ @Override
+ public void removeDataProductFromMetadataSchema(DataProductRemoveFromMetadataSchemaRequest request,
+ StreamObserver<DataProductRemoveFromMetadataSchemaResponse> responseObserver) {
+ // TODO Auto-generated method stub
+ super.removeDataProductFromMetadataSchema(request, responseObserver);
+ }
+
+ @Override
+ public void updateMetadataSchemaField(MetadataSchemaFieldUpdateRequest request,
+ StreamObserver<MetadataSchemaFieldUpdateResponse> responseObserver) {
+ // TODO Auto-generated method stub
+ super.updateMetadataSchemaField(request, responseObserver);
+ }
}
diff --git a/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto b/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto
index 779002c..b1d40dd 100644
--- a/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto
+++ b/data-catalog-api/stubs/src/main/proto/DataCatalogAPI.proto
@@ -64,6 +64,23 @@ message DataProduct {
optional string metadata = 4;
}
+enum FieldValueType {
+ STRING = 0;
+ INTEGER = 1;
+ FLOAT = 2;
+ BOOLEAN = 3;
+ DATESTRING = 4;
+}
+message MetadataSchema {
+ string schema_name = 1;
+}
+message MetadataSchemaField {
+ string schema_name = 1;
+ string field_name = 2;
+ string json_path = 3;
+ FieldValueType value_type = 4;
+}
+
message DataProductCreateRequest {
DataProduct data_product = 1;
}
@@ -88,9 +105,75 @@ message DataProductDeleteRequest {
message DataProductDeleteResponse {
}
+message MetadataSchemaCreateRequest {
+ UserInfo user_info = 1;
+ MetadataSchema metadata_schema = 2;
+}
+message MetadataSchemaCreateResponse {
+ MetadataSchema metadata_schema = 1;
+}
+message MetadataSchemaDeleteRequest {
+ UserInfo user_info = 1;
+ MetadataSchema metadata_schema = 2;
+}
+message MetadataSchemaDeleteResponse {
+}
+message MetadataSchemaFieldCreateRequest {
+ UserInfo user_info = 1;
+ MetadataSchemaField metadata_schema_field = 2;
+}
+message MetadataSchemaFieldCreateResponse {
+ MetadataSchemaField metadata_schema_field = 1;
+}
+message MetadataSchemaFieldUpdateRequest {
+ UserInfo user_info = 1;
+ MetadataSchemaField metadata_schema_field = 2;
+}
+message MetadataSchemaFieldUpdateResponse {
+ MetadataSchemaField metadata_schema_field = 1;
+}
+message MetadataSchemaFieldDeleteRequest {
+ UserInfo user_info = 1;
+ MetadataSchemaField metadata_schema_field = 2;
+}
+message MetadataSchemaFieldDeleteResponse {
+}
+message MetadataSchemaFieldListRequest {
+ UserInfo user_info = 1;
+ string schema_name = 2;
+}
+message MetadataSchemaFieldListResponse {
+ repeated MetadataSchemaField metadata_schema_fields = 1;
+}
+message DataProductAddToMetadataSchemaRequest {
+ UserInfo user_info = 1;
+ string data_product_id = 2;
+ string schema_name = 3;
+}
+message DataProductAddToMetadataSchemaResponse {
+ DataProduct data_product = 1;
+}
+message DataProductRemoveFromMetadataSchemaRequest {
+ UserInfo user_info = 1;
+ string data_product_id = 2;
+ string schema_name = 3;
+}
+message DataProductRemoveFromMetadataSchemaResponse {
+ DataProduct data_product = 1;
+}
+
+
service DataCatalogAPIService {
rpc createDataProduct(DataProductCreateRequest) returns (DataProductCreateResponse){}
rpc updateDataProduct(DataProductUpdateRequest) returns (DataProductUpdateResponse){}
rpc getDataProduct(DataProductGetRequest) returns (DataProductGetResponse){}
rpc deleteDataProduct(DataProductDeleteRequest) returns (DataProductDeleteResponse){}
+ rpc createMetadataSchema(MetadataSchemaCreateRequest) returns (MetadataSchemaCreateResponse){}
+ rpc deleteMetadataSchema(MetadataSchemaCreateRequest) returns (MetadataSchemaCreateResponse){}
+ rpc createMetadataSchemaField(MetadataSchemaFieldCreateRequest) returns (MetadataSchemaFieldCreateResponse){}
+ rpc updateMetadataSchemaField(MetadataSchemaFieldUpdateRequest) returns (MetadataSchemaFieldUpdateResponse){}
+ rpc deleteMetadataSchemaField(MetadataSchemaFieldDeleteRequest) returns (MetadataSchemaFieldDeleteResponse){}
+ rpc getMetadataSchemaFields(MetadataSchemaFieldListRequest) returns (MetadataSchemaFieldListResponse){}
+ rpc addDataProductToMetadataSchema(DataProductAddToMetadataSchemaRequest) returns (DataProductAddToMetadataSchemaResponse){}
+ rpc removeDataProductFromMetadataSchema(DataProductRemoveFromMetadataSchemaRequest) returns (DataProductRemoveFromMetadataSchemaResponse){}
}