You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@texera.apache.org by gi...@apache.org on 2026/08/03 03:16:17 UTC
(texera) 01/01: feat(dataset): add contributor metadata storage and API (#6952)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-6952-f8e58461e6ed805e0bd15b52da946e42692a5ccd
in repository https://gitbox.apache.org/repos/asf/texera.git
commit 74abf0e14a5436066092bd30345e2cc37b3717eb
Author: Xuan Gu <16...@users.noreply.github.com>
AuthorDate: Sun Aug 2 20:16:05 2026 -0700
feat(dataset): add contributor metadata storage and API (#6952)
### What changes were proposed in this PR?
This PR adds contributor metadata to datasets so that dataset authors
and other contributors can be properly acknowledged. Each dataset can
record a list of contributors, where each contributor has a name, a
creator flag, and optional email, affiliation, and free-text comments
fields.
Changes:
- Adds a new `dataset_contributor` table with a foreign key to `dataset`
and cascade deletion in `texera_ddl.sql`, along with the incremental
migration script `sql/updates/30.sql` and changelog entry 30.
- Updates `file-service` so that contributors are:
- returned with the dataset detail response,
- accepted during dataset creation, and
- replaceable through a new `POST /dataset/update/contributors` endpoint
(with `did` in the request body, matching the existing
`update/description` and `update/name` endpoints), protected by the
existing WRITE-access check.
- Validates that each contributor has a name, and that name, email, and
affiliation do not exceed 256 characters.
- Models email, affiliation, and comments as optional in the API:
omitted fields deserialize to null.
Currently, contributors are stored independently for each dataset.
Cross-dataset contributor identity is left as a future extension. The UI
will be added in a follow-up PR.
### Any related issues, documentation, discussions?
Related to #6926
### How was this PR tested?
7 ScalaTest cases were added, covering persisting contributors at
dataset creation, replacing and clearing the list, validation failures
(missing name, over-length fields), the WRITE-access check, and cascade
deletion when the dataset is removed.
### Was this PR authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Claude Fable 5)
Co-authored-by: Claude Fable 5 <no...@anthropic.com>
---
.../texera/service/resource/DatasetResource.scala | 105 +++++++++-
.../service/resource/DatasetResourceSpec.scala | 225 +++++++++++++++++++++
sql/changelog.xml | 5 +
sql/texera_ddl.sql | 14 ++
sql/updates/30.sql | 38 ++++
5 files changed, 383 insertions(+), 4 deletions(-)
diff --git a/file-service/src/main/scala/org/apache/texera/service/resource/DatasetResource.scala b/file-service/src/main/scala/org/apache/texera/service/resource/DatasetResource.scala
index c376e4ce04..ce872f4520 100644
--- a/file-service/src/main/scala/org/apache/texera/service/resource/DatasetResource.scala
+++ b/file-service/src/main/scala/org/apache/texera/service/resource/DatasetResource.scala
@@ -34,6 +34,7 @@ import org.apache.texera.dao.SqlServer
import org.apache.texera.dao.SqlServer.withTransaction
import org.apache.texera.dao.jooq.generated.enums.PrivilegeEnum
import org.apache.texera.dao.jooq.generated.tables.Dataset.DATASET
+import org.apache.texera.dao.jooq.generated.tables.DatasetContributor.DATASET_CONTRIBUTOR
import org.apache.texera.dao.jooq.generated.tables.DatasetUserAccess.DATASET_USER_ACCESS
import org.apache.texera.dao.jooq.generated.tables.DatasetVersion.DATASET_VERSION
import org.apache.texera.dao.jooq.generated.tables.User.USER
@@ -175,12 +176,74 @@ object DatasetResource {
normalized
}
+ /**
+ * Helper function to get the contributors using the did
+ */
+ def getContributorsByDid(ctx: DSLContext, did: Integer): List[Contributor] = {
+ ctx
+ .selectFrom(DATASET_CONTRIBUTOR)
+ .where(DATASET_CONTRIBUTOR.DID.eq(did))
+ .fetch()
+ .asScala
+ .toList
+ .map { record =>
+ Contributor(
+ name = record.getName,
+ creator = record.getCreator,
+ affiliation = record.getAffiliation,
+ email = record.getEmail,
+ comments = record.getComments
+ )
+ }
+ }
+
+ /**
+ * Helper function to insert the contributors of a dataset in one batch
+ */
+ def insertContributors(ctx: DSLContext, did: Integer, contributors: List[Contributor]): Unit = {
+ val records = contributors.map { contributor =>
+ if (contributor == null || contributor.name == null || contributor.name.trim.isEmpty) {
+ throw new BadRequestException("Each contributor must have a name")
+ }
+ if (
+ contributor.name.length > 256 ||
+ Option(contributor.email).exists(_.length > 256) ||
+ Option(contributor.affiliation).exists(_.length > 256)
+ ) {
+ throw new BadRequestException("Contributor fields must not exceed 256 characters")
+ }
+ val record = ctx.newRecord(DATASET_CONTRIBUTOR)
+ record.setDid(did)
+ record.setName(contributor.name)
+ record.setCreator(contributor.creator)
+ record.setAffiliation(contributor.affiliation)
+ record.setEmail(contributor.email)
+ record.setComments(contributor.comments)
+ record
+ }
+ ctx.batchInsert(records.asJava).execute()
+ }
+
+ case class Contributor(
+ name: String,
+ creator: Boolean = false,
+ affiliation: String = null,
+ email: String = null,
+ comments: String = null
+ )
+
+ case class DatasetContributorsModification(
+ did: Integer,
+ contributors: Option[List[Contributor]] = None
+ )
+
case class DashboardDataset(
dataset: Dataset,
ownerEmail: String,
accessPrivilege: EnumType,
isOwner: Boolean,
- size: Long
+ size: Long,
+ contributors: List[Contributor] = Nil
)
case class DashboardDatasetVersion(
@@ -192,7 +255,8 @@ object DatasetResource {
datasetName: String,
datasetDescription: String,
isDatasetPublic: Boolean,
- isDatasetDownloadable: Boolean
+ isDatasetDownloadable: Boolean,
+ contributors: Option[List[Contributor]] = None
)
case class Diff(
@@ -257,7 +321,8 @@ class DatasetResource extends LazyLogging {
isOwner,
withLakeFSErrorHandling(s"retrieving the size of dataset '${targetDataset.getName}'") {
LakeFSStorageClient.retrieveRepositorySize(targetDataset.getRepositoryName)
- }
+ },
+ contributors = DatasetResource.getContributorsByDid(ctx, did)
)
}
@@ -309,6 +374,9 @@ class DatasetResource extends LazyLogging {
.fetchOne()
}
+ val savedContributors = request.contributors.getOrElse(Nil)
+ DatasetResource.insertContributors(ctx, createdDataset.getDid, savedContributors)
+
// Initialize the repository in LakeFS
val repositoryName = s"dataset-${createdDataset.getDid}"
try {
@@ -347,7 +415,8 @@ class DatasetResource extends LazyLogging {
user.getEmail,
PrivilegeEnum.WRITE,
isOwner = true,
- 0
+ 0,
+ savedContributors
)
}
}
@@ -494,6 +563,34 @@ class DatasetResource extends LazyLogging {
}
}
+ @POST
+ @Consumes(Array(MediaType.APPLICATION_JSON))
+ @Produces(Array(MediaType.APPLICATION_JSON))
+ @RolesAllowed(Array("REGULAR", "ADMIN"))
+ @Path("/update/contributors")
+ def updateDatasetContributors(
+ modificator: DatasetContributorsModification,
+ @Auth user: SessionUser
+ ): Response = {
+ withTransaction(context) { ctx =>
+ if (!userHasWriteAccess(ctx, modificator.did, user.getUid)) {
+ throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE)
+ }
+
+ ctx
+ .delete(DATASET_CONTRIBUTOR)
+ .where(DATASET_CONTRIBUTOR.DID.eq(modificator.did))
+ .execute()
+ DatasetResource.insertContributors(
+ ctx,
+ modificator.did,
+ modificator.contributors.getOrElse(Nil)
+ )
+
+ Response.ok().build()
+ }
+ }
+
@POST
@Consumes(Array(MediaType.APPLICATION_JSON))
@Produces(Array(MediaType.APPLICATION_JSON))
diff --git a/file-service/src/test/scala/org/apache/texera/service/resource/DatasetResourceSpec.scala b/file-service/src/test/scala/org/apache/texera/service/resource/DatasetResourceSpec.scala
index fb27fa6731..7ca1e9429c 100644
--- a/file-service/src/test/scala/org/apache/texera/service/resource/DatasetResourceSpec.scala
+++ b/file-service/src/test/scala/org/apache/texera/service/resource/DatasetResourceSpec.scala
@@ -343,6 +343,40 @@ class DatasetResourceSpec
dashboardDataset.dataset.getIsDownloadable shouldBe false
}
+ it should "persist contributors and return them in the response" in {
+ val contributors = List(
+ DatasetResource.Contributor(
+ name = "test1",
+ creator = true,
+ affiliation = "Test Lab A",
+ email = "contributor-a@test.com",
+ comments = "collected the data"
+ ),
+ DatasetResource.Contributor(
+ name = "test2",
+ creator = false,
+ affiliation = "Test Lab B",
+ email = "contributor-b@test.com",
+ comments = null
+ )
+ )
+ val createDatasetRequest = DatasetResource.CreateDatasetRequest(
+ datasetName = "contributor-ds",
+ datasetDescription = "dataset with contributors",
+ isDatasetPublic = false,
+ isDatasetDownloadable = true,
+ contributors = Some(contributors)
+ )
+
+ val createdDataset = datasetResource.createDataset(createDatasetRequest, sessionUser)
+
+ createdDataset.contributors should contain theSameElementsAs contributors
+ DatasetResource.getContributorsByDid(
+ getDSLContext,
+ createdDataset.dataset.getDid
+ ) should contain theSameElementsAs contributors
+ }
+
it should "delete dataset successfully if user owns it" in {
val dataset = new Dataset
dataset.setName("delete-ds")
@@ -404,6 +438,197 @@ class DatasetResourceSpec
dashboardDataset.size should be >= 0L
}
+ "updateDatasetContributors" should "replace the contributor list of the dataset" in {
+ val initial = List(
+ DatasetResource.Contributor(
+ "test1",
+ creator = true,
+ "Test Lab A",
+ "contributor-a@test.com",
+ "initial"
+ )
+ )
+ val createdDataset = datasetResource.createDataset(
+ DatasetResource.CreateDatasetRequest(
+ datasetName = "contributor-update-ds",
+ datasetDescription = "dataset for contributor update",
+ isDatasetPublic = false,
+ isDatasetDownloadable = true,
+ contributors = Some(initial)
+ ),
+ sessionUser
+ )
+ val did = createdDataset.dataset.getDid
+
+ val replacement = List(
+ DatasetResource
+ .Contributor("test2", creator = false, "Test Lab C", "contributor-c@test.com", "curation"),
+ DatasetResource.Contributor(
+ "test3",
+ creator = true,
+ "Test Lab D",
+ "contributor-d@test.com",
+ "analysis"
+ )
+ )
+ val response = datasetResource.updateDatasetContributors(
+ DatasetResource.DatasetContributorsModification(did, Some(replacement)),
+ sessionUser
+ )
+
+ response.getStatus shouldEqual 200
+ DatasetResource.getContributorsByDid(
+ getDSLContext,
+ did
+ ) should contain theSameElementsAs replacement
+ }
+
+ it should "clear all contributors when given an empty list" in {
+ val createdDataset = datasetResource.createDataset(
+ DatasetResource.CreateDatasetRequest(
+ datasetName = "contributor-clear-ds",
+ datasetDescription = "dataset for contributor clear test",
+ isDatasetPublic = false,
+ isDatasetDownloadable = true,
+ contributors = Some(
+ List(
+ DatasetResource
+ .Contributor("test1", creator = true, "Test Lab A", "contributor-a@test.com", null),
+ DatasetResource
+ .Contributor("test2", creator = false, "Test Lab B", "contributor-b@test.com", null)
+ )
+ )
+ ),
+ sessionUser
+ )
+ val did = createdDataset.dataset.getDid
+ DatasetResource.getContributorsByDid(getDSLContext, did) should have size 2
+
+ val response = datasetResource.updateDatasetContributors(
+ DatasetResource.DatasetContributorsModification(did, Some(Nil)),
+ sessionUser
+ )
+
+ response.getStatus shouldEqual 200
+ DatasetResource.getContributorsByDid(getDSLContext, did) shouldBe empty
+ }
+
+ it should "reject a contributor without a name" in {
+ val createdDataset = datasetResource.createDataset(
+ DatasetResource.CreateDatasetRequest(
+ datasetName = "contributor-noname-ds",
+ datasetDescription = "dataset for contributor validation test",
+ isDatasetPublic = false,
+ isDatasetDownloadable = true
+ ),
+ sessionUser
+ )
+ val did = createdDataset.dataset.getDid
+
+ assertThrows[BadRequestException] {
+ datasetResource.updateDatasetContributors(
+ DatasetResource.DatasetContributorsModification(
+ did,
+ Some(
+ List(
+ DatasetResource
+ .Contributor(null, creator = false, "Test Lab A", "contributor-x@test.com", null)
+ )
+ )
+ ),
+ sessionUser
+ )
+ }
+ DatasetResource.getContributorsByDid(getDSLContext, did) shouldBe empty
+ }
+
+ it should "reject contributor fields longer than 256 characters" in {
+ val createdDataset = datasetResource.createDataset(
+ DatasetResource.CreateDatasetRequest(
+ datasetName = "contributor-toolong-ds",
+ datasetDescription = "dataset for contributor length test",
+ isDatasetPublic = false,
+ isDatasetDownloadable = true
+ ),
+ sessionUser
+ )
+
+ assertThrows[BadRequestException] {
+ datasetResource.updateDatasetContributors(
+ DatasetResource.DatasetContributorsModification(
+ createdDataset.dataset.getDid,
+ Some(
+ List(
+ DatasetResource
+ .Contributor(
+ "A" * 257,
+ creator = false,
+ "Test Lab A",
+ "contributor-x@test.com",
+ null
+ )
+ )
+ )
+ ),
+ sessionUser
+ )
+ }
+ }
+
+ it should "refuse to update contributors without write access" in {
+ val createdDataset = datasetResource.createDataset(
+ DatasetResource.CreateDatasetRequest(
+ datasetName = "contributor-forbidden-ds",
+ datasetDescription = "dataset for contributor permission test",
+ isDatasetPublic = true,
+ isDatasetDownloadable = true
+ ),
+ sessionUser
+ )
+ val did = createdDataset.dataset.getDid
+
+ assertThrows[ForbiddenException] {
+ datasetResource.updateDatasetContributors(
+ DatasetResource.DatasetContributorsModification(
+ did,
+ Some(
+ List(
+ DatasetResource
+ .Contributor("test1", creator = false, "Test Lab E", "contributor-e@test.com", null)
+ )
+ )
+ ),
+ multipartNoWriteSessionUser
+ )
+ }
+
+ DatasetResource.getContributorsByDid(getDSLContext, did) shouldBe empty
+ }
+
+ it should "remove contributors when their dataset is deleted" in {
+ val createdDataset = datasetResource.createDataset(
+ DatasetResource.CreateDatasetRequest(
+ datasetName = "contributor-cascade-ds",
+ datasetDescription = "dataset for contributor cascade test",
+ isDatasetPublic = false,
+ isDatasetDownloadable = true,
+ contributors = Some(
+ List(
+ DatasetResource
+ .Contributor("test1", creator = true, "Test Lab A", "contributor-f@test.com", null)
+ )
+ )
+ ),
+ sessionUser
+ )
+ val did = createdDataset.dataset.getDid
+ DatasetResource.getContributorsByDid(getDSLContext, did) should have size 1
+
+ datasetResource.deleteDataset(did, sessionUser).getStatus shouldEqual 200
+
+ DatasetResource.getContributorsByDid(getDSLContext, did) shouldBe empty
+ }
+
"findExistingUploadFiles" should "match committed and staged files by path and size" in {
val repoName = s"existing-upload-${System.nanoTime()}"
val dataset = new Dataset
diff --git a/sql/changelog.xml b/sql/changelog.xml
index 2cec53da2f..0288dbd6b8 100644
--- a/sql/changelog.xml
+++ b/sql/changelog.xml
@@ -58,6 +58,11 @@
<sqlFile path="sql/updates/29.sql"/>
</changeSet>
+ <!-- Add dataset_contributor table (contributor metadata for datasets) -->
+ <changeSet id="30" author="xuang7">
+ <sqlFile path="sql/updates/30.sql"/>
+ </changeSet>
+
<!-- example changeSet
<changeSet id="1" author="author">
<sqlFile path="sql/updates/1.sql"/>
diff --git a/sql/texera_ddl.sql b/sql/texera_ddl.sql
index 4132aba32c..982f784a47 100644
--- a/sql/texera_ddl.sql
+++ b/sql/texera_ddl.sql
@@ -65,6 +65,7 @@ DROP TABLE IF EXISTS dataset_upload_session_part CASCADE;
DROP TABLE IF EXISTS dataset CASCADE;
DROP TABLE IF EXISTS dataset_user_access CASCADE;
DROP TABLE IF EXISTS dataset_version CASCADE;
+DROP TABLE IF EXISTS dataset_contributor CASCADE;
DROP TABLE IF EXISTS public_project CASCADE;
DROP TABLE IF EXISTS project_user_access CASCADE;
DROP TABLE IF EXISTS workflow_user_likes CASCADE;
@@ -314,6 +315,19 @@ CREATE TABLE IF NOT EXISTS dataset_version
FOREIGN KEY (did) REFERENCES dataset(did) ON DELETE CASCADE
);
+-- dataset_contributor
+CREATE TABLE IF NOT EXISTS dataset_contributor
+(
+ cid SERIAL PRIMARY KEY,
+ did INT NOT NULL,
+ name VARCHAR(256) NOT NULL,
+ creator BOOLEAN NOT NULL DEFAULT FALSE,
+ email VARCHAR(256),
+ affiliation VARCHAR(256),
+ comments TEXT,
+ FOREIGN KEY (did) REFERENCES dataset(did) ON DELETE CASCADE
+ );
+
CREATE TABLE IF NOT EXISTS dataset_upload_session
(
did INT NOT NULL,
diff --git a/sql/updates/30.sql b/sql/updates/30.sql
new file mode 100644
index 0000000000..e3d06ce53a
--- /dev/null
+++ b/sql/updates/30.sql
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+\c texera_db
+
+SET search_path TO texera_db;
+
+BEGIN;
+
+CREATE TABLE IF NOT EXISTS dataset_contributor
+(
+ cid SERIAL PRIMARY KEY,
+ did INT NOT NULL,
+ name VARCHAR(256) NOT NULL,
+ creator BOOLEAN NOT NULL DEFAULT FALSE,
+ email VARCHAR(256),
+ affiliation VARCHAR(256),
+ comments TEXT,
+ FOREIGN KEY (did) REFERENCES dataset(did) ON DELETE CASCADE
+);
+
+COMMIT;