You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@texera.apache.org by ku...@apache.org on 2025/10/27 20:16:01 UTC
(texera) branch chris-big-object-version-0 updated (768d34a377 -> cea73acbb6)
This is an automated email from the ASF dual-hosted git repository.
kunwp1 pushed a change to branch chris-big-object-version-0
in repository https://gitbox.apache.org/repos/asf/texera.git
from 768d34a377 Big Object for Java Native Operators
new b8129d1d26 chore(deps): bump brace-expansion from 1.1.11 to 1.1.12 in /core/gui (#3798)
new 611f508686 refactor(frontend): Simplify onClickToggleGrids logic (#3896)
new c2c59409e3 Big Object Read for Python UDF Operators
new 7d98b74e72 Big Object Read for R UDF Operators
new 038ebb7256 Add operator id to the table
new baa5504e1b create() function for python udf
new cea73acbb6 create() API for python + cleanup the parameters
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
amber/operator-requirements.txt | 1 +
.../control/initialize_executor_handler.py | 5 +
.../python/core/architecture/managers/context.py | 13 ++
.../architecture/managers/execution_context.py | 62 ++++++
.../core/architecture/packaging/output_manager.py | 12 +-
.../src/main/python/core/models/RTableExecutor.py | 108 ++++++++-
.../src/main/python/core/models/RTupleExecutor.py | 30 ++-
.../main/python/core/models/big_object_manager.R | 202 +++++++++++++++++
.../main/python/core/models/big_object_pointer.R | 51 +++++
amber/src/main/python/core/models/r_utils.py | 18 ++
.../src/main/python/core/models/schema/__init__.py | 2 +
.../python/core/models/schema/attribute_type.py | 6 +
.../core/models/schema/big_object_pointer.py | 65 ++++++
amber/src/main/python/core/models/schema/schema.py | 36 ++-
amber/src/main/python/core/models/tuple.py | 18 +-
.../main/python/core/runnables/data_processor.py | 5 +
.../main/python/core/storage/document_factory.py | 4 +-
.../python/core/storage/iceberg/iceberg_utils.py | 153 ++++++++++++-
.../src/main/python/core/storage/storage_config.py | 34 ++-
.../amber/engine/architecture/rpc/__init__.py | 1 +
amber/src/main/python/pytexera/__init__.py | 4 +
.../python/pytexera/storage/big_object_manager.py | 245 +++++++++++++++++++++
amber/src/main/python/texera_run_python_worker.py | 14 ++
.../pythonworker/PythonWorkflowWorker.scala | 9 +-
.../InitializeExecutorHandler.scala | 1 +
bin/deployment/computing-unit-master.dockerfile | 4 +-
common/workflow-core/build.sbt | 4 +-
.../amber/core/executor/ExecutionContext.scala | 23 +-
.../amber/core/tuple/AttributeTypeUtils.scala | 2 +-
.../scala/org/apache/amber/util/ArrowUtils.scala | 62 ++++--
.../texera/service/util/BigObjectManager.scala | 61 ++---
.../source/scan/FileScanSourceOpExec.scala | 9 +-
.../app/workspace/component/menu/menu.component.ts | 4 +-
.../workflow-editor/workflow-editor.component.ts | 21 --
.../workflow-graph/model/joint-graph-wrapper.ts | 16 --
frontend/yarn.lock | 6 +-
sql/texera_ddl.sql | 1 +
sql/updates/big_object.sql | 1 +
38 files changed, 1167 insertions(+), 146 deletions(-)
create mode 100644 amber/src/main/python/core/architecture/managers/execution_context.py
create mode 100644 amber/src/main/python/core/models/big_object_manager.R
create mode 100644 amber/src/main/python/core/models/big_object_pointer.R
create mode 100644 amber/src/main/python/core/models/schema/big_object_pointer.py
create mode 100644 amber/src/main/python/pytexera/storage/big_object_manager.py
(texera) 01/07: chore(deps): bump brace-expansion from 1.1.11 to 1.1.12 in /core/gui (#3798)
Posted by ku...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kunwp1 pushed a commit to branch chris-big-object-version-0
in repository https://gitbox.apache.org/repos/asf/texera.git
commit b8129d1d260e4f631422d9e74edf36fd48427e7e
Author: dependabot[bot] <49...@users.noreply.github.com>
AuthorDate: Tue Oct 14 19:01:18 2025 +0000
chore(deps): bump brace-expansion from 1.1.11 to 1.1.12 in /core/gui (#3798)
Bumps [brace-expansion](https://github.com/juliangruber/brace-expansion)
from 1.1.11 to 1.1.12.
<details>
<summary>Release notes</summary>
<p><em>Sourced from <a
href="https://github.com/juliangruber/brace-expansion/releases">brace-expansion's
releases</a>.</em></p>
<blockquote>
<h2>v1.1.12</h2>
<ul>
<li>pkg: publish on tag 1.x c460dbd</li>
<li>fmt ccb8ac6</li>
<li>Fix potential ReDoS Vulnerability or Inefficient Regular Expression
(<a
href="https://redirect.github.com/juliangruber/brace-expansion/issues/65">#65</a>)
c3c73c8</li>
</ul>
<hr />
<p><a
href="https://github.com/juliangruber/brace-expansion/compare/v1.1.11...v1.1.12">https://github.com/juliangruber/brace-expansion/compare/v1.1.11...v1.1.12</a></p>
</blockquote>
</details>
<details>
<summary>Commits</summary>
<ul>
<li><a
href="https://github.com/juliangruber/brace-expansion/commit/44f33b47c5c6a965d507421af43e86cf5971d711"><code>44f33b4</code></a>
1.1.12</li>
<li><a
href="https://github.com/juliangruber/brace-expansion/commit/c460dbd68e428d147b2080622d8ce126c7a08570"><code>c460dbd</code></a>
pkg: publish on tag 1.x</li>
<li><a
href="https://github.com/juliangruber/brace-expansion/commit/ccb8ac6d4292b7661b677fe048ba6690c877f51f"><code>ccb8ac6</code></a>
fmt</li>
<li><a
href="https://github.com/juliangruber/brace-expansion/commit/c3c73c8b088defc70851843be88ccc3af08e7217"><code>c3c73c8</code></a>
Fix potential ReDoS Vulnerability or Inefficient Regular Expression (<a
href="https://redirect.github.com/juliangruber/brace-expansion/issues/65">#65</a>)</li>
<li>See full diff in <a
href="https://github.com/juliangruber/brace-expansion/compare/1.1.11...v1.1.12">compare
view</a></li>
</ul>
</details>
<br />
[](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores)
Dependabot will resolve any conflicts with this PR as long as you don't
alter it yourself. You can also trigger a rebase manually by commenting
`@dependabot rebase`.
[//]: # (dependabot-automerge-start)
[//]: # (dependabot-automerge-end)
---
<details>
<summary>Dependabot commands and options</summary>
<br />
You can trigger Dependabot actions by commenting on this PR:
- `@dependabot rebase` will rebase this PR
- `@dependabot recreate` will recreate this PR, overwriting any edits
that have been made to it
- `@dependabot merge` will merge this PR after your CI passes on it
- `@dependabot squash and merge` will squash and merge this PR after
your CI passes on it
- `@dependabot cancel merge` will cancel a previously requested merge
and block automerging
- `@dependabot reopen` will reopen this PR if it is closed
- `@dependabot close` will close this PR and stop Dependabot recreating
it. You can achieve the same result by closing it manually
- `@dependabot show <dependency name> ignore conditions` will show all
of the ignore conditions of the specified dependency
- `@dependabot ignore this major version` will close this PR and stop
Dependabot creating any more for this major version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this minor version` will close this PR and stop
Dependabot creating any more for this minor version (unless you reopen
the PR or upgrade to it yourself)
- `@dependabot ignore this dependency` will close this PR and stop
Dependabot creating any more for this dependency (unless you reopen the
PR or upgrade to it yourself)
You can disable automated security fix PRs for this repo from the
[Security Alerts page](https://github.com/apache/texera/network/alerts).
</details>
Signed-off-by: dependabot[bot] <su...@github.com>
Co-authored-by: dependabot[bot] <49...@users.noreply.github.com>
Co-authored-by: Xiaozhen Liu <xi...@uci.edu>
Co-authored-by: Xinyuan Lin <xi...@uci.edu>
---
frontend/yarn.lock | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/frontend/yarn.lock b/frontend/yarn.lock
index 43ae178efa..ff9e9634f7 100644
--- a/frontend/yarn.lock
+++ b/frontend/yarn.lock
@@ -7085,12 +7085,12 @@ __metadata:
linkType: hard
"brace-expansion@npm:^1.1.7":
- version: 1.1.11
- resolution: "brace-expansion@npm:1.1.11"
+ version: 1.1.12
+ resolution: "brace-expansion@npm:1.1.12"
dependencies:
balanced-match: "npm:^1.0.0"
concat-map: "npm:0.0.1"
- checksum: 10c0/695a56cd058096a7cb71fb09d9d6a7070113c7be516699ed361317aca2ec169f618e28b8af352e02ab4233fb54eb0168460a40dc320bab0034b36ab59aaad668
+ checksum: 10c0/975fecac2bb7758c062c20d0b3b6288c7cc895219ee25f0a64a9de662dbac981ff0b6e89909c3897c1f84fa353113a721923afdec5f8b2350255b097f12b1f73
languageName: node
linkType: hard
(texera) 02/07: refactor(frontend): Simplify onClickToggleGrids logic (#3896)
Posted by ku...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kunwp1 pushed a commit to branch chris-big-object-version-0
in repository https://gitbox.apache.org/repos/asf/texera.git
commit 611f50868691ddbaa4d76bae83462aca39feee39
Author: Xinyuan Lin <xi...@uci.edu>
AuthorDate: Tue Oct 14 13:09:30 2025 -0700
refactor(frontend): Simplify onClickToggleGrids logic (#3896)
---
.../app/workspace/component/menu/menu.component.ts | 4 +++-
.../workflow-editor/workflow-editor.component.ts | 21 ---------------------
.../workflow-graph/model/joint-graph-wrapper.ts | 16 ----------------
3 files changed, 3 insertions(+), 38 deletions(-)
diff --git a/frontend/src/app/workspace/component/menu/menu.component.ts b/frontend/src/app/workspace/component/menu/menu.component.ts
index b26d1e82b7..4b5c3835f5 100644
--- a/frontend/src/app/workspace/component/menu/menu.component.ts
+++ b/frontend/src/app/workspace/component/menu/menu.component.ts
@@ -462,7 +462,9 @@ export class MenuComponent implements OnInit, OnDestroy {
* This option is only for the current session and will be cleared on refresh.
*/
public onClickToggleGrids(): void {
- this.workflowActionService.getJointGraphWrapper().toggleGrids();
+ const paper = this.workflowActionService.getJointGraphWrapper().mainPaper;
+ const current = paper.options.gridSize || 1;
+ paper.setGridSize((current % 2) + 1);
}
/**
diff --git a/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.ts b/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.ts
index 6f53fa7371..a334b48603 100644
--- a/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.ts
+++ b/frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.ts
@@ -87,7 +87,6 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy
editorWrapper!: HTMLElement;
paper!: joint.dia.Paper;
private interactive: boolean = true;
- private gridOn: boolean = false;
private _onProcessKeyboardActionObservable: Subject<void> = new Subject();
private wrapper;
private currentOpenedOperatorID: string | null = null;
@@ -169,7 +168,6 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy
this.handleElementCut();
this.handleElementPaste();
this.handleLinkCursorHover();
- this.handleGridsToggle();
if (this.config.env.linkBreakpointEnabled && this.workflowActionService.getHighlightingEnabled()) {
this.handleLinkBreakpoint();
}
@@ -1246,25 +1244,6 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy
return this.workflowActionService.getTexeraGraph().getOperator(operatorID).outputPorts.length == 0;
}
- /**
- * This function handles the event stream from jointGraph to toggle the grids in jointPaper on or off.
- * @private
- */
- private handleGridsToggle(): void {
- this.wrapper
- .getJointPaperGridsToggleStream()
- .pipe(untilDestroyed(this))
- .subscribe(() => {
- if (this.gridOn) {
- this.paper.setGridSize(1);
- this.gridOn = false;
- } else {
- this.paper.setGridSize(2);
- this.gridOn = true;
- }
- });
- }
-
/**
* Handles mouse events to enable shared cursor.
*/
diff --git a/frontend/src/app/workspace/service/workflow-graph/model/joint-graph-wrapper.ts b/frontend/src/app/workspace/service/workflow-graph/model/joint-graph-wrapper.ts
index 9793792c9b..a519bac1a2 100644
--- a/frontend/src/app/workspace/service/workflow-graph/model/joint-graph-wrapper.ts
+++ b/frontend/src/app/workspace/service/workflow-graph/model/joint-graph-wrapper.ts
@@ -140,8 +140,6 @@ export class JointGraphWrapper {
private workflowEditorZoomSubject: Subject<number> = new Subject<number>();
// event stream of restoring zoom / offset default of the jointJS paper
private restorePaperOffsetSubject: Subject<void> = new Subject<void>();
- // event stream to toggle the jointPaper grids on or off.
- private jointPaperGridsToggleStream = new Subject<void>();
// event stream of showing the breakpoint button of a link
private jointLinkBreakpointShowStream = new Subject<linkIDType>();
@@ -884,20 +882,6 @@ export class JointGraphWrapper {
return JointGraphContext;
}
- /**
- * Returns an observable to indicate a toggle of grids has happened, and lets workflow editor to handle the state.
- */
- public getJointPaperGridsToggleStream(): Observable<void> {
- return this.jointPaperGridsToggleStream.asObservable();
- }
-
- /**
- * Triggers a toggle of whether to show grids in jointPaper.
- */
- public toggleGrids() {
- this.jointPaperGridsToggleStream.next();
- }
-
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// Below are methods for coeditor-presence. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
(texera) 05/07: Add operator id to the table
Posted by ku...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kunwp1 pushed a commit to branch chris-big-object-version-0
in repository https://gitbox.apache.org/repos/asf/texera.git
commit 038ebb725649e46142348a536441aed79c51bd92
Author: Kunwoo Park <ku...@uci.edu>
AuthorDate: Mon Oct 27 10:27:46 2025 -0700
Add operator id to the table
---
.../main/python/core/models/big_object_manager.R | 119 +++++++++++++++++++--
.../InitializeExecutorHandler.scala | 1 +
.../amber/core/executor/ExecutionContext.scala | 23 +++-
.../source/scan/FileScanSourceOpExec.scala | 8 +-
sql/texera_ddl.sql | 1 +
sql/updates/big_object.sql | 1 +
6 files changed, 141 insertions(+), 12 deletions(-)
diff --git a/amber/src/main/python/core/models/big_object_manager.R b/amber/src/main/python/core/models/big_object_manager.R
index c365c11141..9e33f23149 100644
--- a/amber/src/main/python/core/models/big_object_manager.R
+++ b/amber/src/main/python/core/models/big_object_manager.R
@@ -27,17 +27,40 @@ if (!require("aws.s3", quietly = TRUE)) {
# BigObjectStream Reference Class
# Provides stream-like access to big object content
+#
+# Uses S3 streaming connection (downloads on-demand, not upfront!)
+# Memory usage: O(1) - only current chunk in memory
BigObjectStream <- setRefClass("BigObjectStream",
fields = list(conn = "ANY", uri = "character", is_closed = "logical"),
methods = list(
- initialize = function(raw_bytes, uri_val) {
- conn <<- rawConnection(raw_bytes, open = "rb")
+ initialize = function(s3_conn, uri_val) {
+ # Store the S3 connection (NOT raw bytes!)
+ # This enables true streaming (downloads on-demand)
+ conn <<- s3_conn
uri <<- uri_val
is_closed <<- FALSE
},
read = function(n = -1L) {
if (is_closed) stop("Stream is closed")
- readBin(conn, "raw", if (n == -1L) 1e9 else n)
+
+ if (n == -1L) {
+ # Read all remaining data in chunks to avoid memory spike
+ # Downloads incrementally from S3 (not all at once!)
+ result <- raw(0)
+ chunk_size <- 10 * 1024 * 1024 # 10MB chunks
+ repeat {
+ chunk <- tryCatch(
+ readBin(conn, "raw", chunk_size),
+ error = function(e) raw(0)
+ )
+ if (length(chunk) == 0) break
+ result <- c(result, chunk)
+ }
+ result
+ } else {
+ # Read exactly n bytes (downloads only n bytes from S3!)
+ readBin(conn, "raw", n)
+ }
},
close = function() {
if (!is_closed) {
@@ -73,6 +96,9 @@ BigObjectManager <- list(
parts <- strsplit(sub("^s3://", "", uri), "/", fixed = TRUE)[[1]]
if (length(parts) < 2) stop(paste("Invalid S3 URI format:", uri))
+ bucket <- parts[1]
+ key <- paste(parts[-1], collapse = "/")
+
# Configure S3 credentials from environment variables
Sys.setenv(
AWS_ACCESS_KEY_ID = Sys.getenv("STORAGE_S3_AUTH_USERNAME", "texera_minio"),
@@ -81,19 +107,96 @@ BigObjectManager <- list(
AWS_DEFAULT_REGION = Sys.getenv("STORAGE_S3_REGION", "us-west-2")
)
- # Fetch object from S3
- raw_bytes <- tryCatch(
- aws.s3::get_object(
+ # Create TRUE STREAMING connection to S3
+ # This does NOT download the file - downloads happen on-demand as you read()!
+ message("BigObjectManager: Opening streaming connection for: ", uri)
+ start_time <- Sys.time()
+
+ s3_conn <- tryCatch(
+ aws.s3::s3connection(
+ object = key,
+ bucket = bucket,
+ region = Sys.getenv("AWS_DEFAULT_REGION"),
+ base_url = Sys.getenv("AWS_S3_ENDPOINT"),
+ use_https = grepl("^https://", Sys.getenv("AWS_S3_ENDPOINT"))
+ ),
+ error = function(e) stop(paste("Failed to open streaming connection for", uri, ":", conditionMessage(e)))
+ )
+
+ open_time <- Sys.time()
+ message(sprintf("BigObjectManager: Streaming connection established in %.2f seconds",
+ as.numeric(difftime(open_time, start_time, units = "secs"))))
+ message("BigObjectManager: Data will be downloaded on-demand as you read() - O(1) memory!")
+
+ BigObjectStream$new(s3_conn, uri)
+ },
+
+ # Fast path for reading RDS files - downloads directly to disk, skips memory
+ readRDS = function(pointer_or_uri) {
+ # Extract from list if needed (for backward compatibility)
+ if (is.list(pointer_or_uri) && length(pointer_or_uri) == 1) {
+ pointer_or_uri <- pointer_or_uri[[1]]
+ }
+
+ # Get URI string
+ uri <- if (inherits(pointer_or_uri, "BigObjectPointer")) {
+ pointer_or_uri$uri
+ } else if (is.character(pointer_or_uri)) {
+ pointer_or_uri
+ } else {
+ stop("Expected BigObjectPointer or character URI")
+ }
+
+ if (!grepl("^s3://", uri)) stop(paste("Invalid S3 URI:", uri))
+
+ # Parse s3://bucket/key
+ parts <- strsplit(sub("^s3://", "", uri), "/", fixed = TRUE)[[1]]
+ if (length(parts) < 2) stop(paste("Invalid S3 URI format:", uri))
+
+ # Configure S3 credentials from environment variables
+ Sys.setenv(
+ AWS_ACCESS_KEY_ID = Sys.getenv("STORAGE_S3_AUTH_USERNAME", "texera_minio"),
+ AWS_SECRET_ACCESS_KEY = Sys.getenv("STORAGE_S3_AUTH_PASSWORD", "password"),
+ AWS_S3_ENDPOINT = Sys.getenv("STORAGE_S3_ENDPOINT", "localhost:9000"),
+ AWS_DEFAULT_REGION = Sys.getenv("STORAGE_S3_REGION", "us-west-2")
+ )
+
+ message("BigObjectManager: Fast readRDS for: ", uri)
+ start_time <- Sys.time()
+
+ # Download directly to temp file (skips loading into memory!)
+ temp_file <- tempfile(fileext = ".rds")
+ tryCatch(
+ aws.s3::save_object(
object = paste(parts[-1], collapse = "/"),
bucket = parts[1],
+ file = temp_file,
region = Sys.getenv("AWS_DEFAULT_REGION"),
base_url = Sys.getenv("AWS_S3_ENDPOINT"),
use_https = grepl("^https://", Sys.getenv("AWS_S3_ENDPOINT"))
),
- error = function(e) stop(paste("Failed to open", uri, ":", conditionMessage(e)))
+ error = function(e) stop(paste("Failed to download", uri, ":", conditionMessage(e)))
)
- BigObjectStream$new(raw_bytes, uri)
+ download_time <- Sys.time()
+ file_size_gb <- file.info(temp_file)$size / 1e9
+ message(sprintf("BigObjectManager: Downloaded %.2f GB to disk in %.2f seconds",
+ file_size_gb,
+ as.numeric(difftime(download_time, start_time, units = "secs"))))
+
+ # Read RDS from temp file
+ result <- base::readRDS(temp_file)
+ read_time <- Sys.time()
+ message(sprintf("BigObjectManager: readRDS() took %.2f seconds",
+ as.numeric(difftime(read_time, download_time, units = "secs"))))
+
+ unlink(temp_file) # Clean up
+
+ total_time <- Sys.time()
+ message(sprintf("BigObjectManager: Total time: %.2f seconds",
+ as.numeric(difftime(total_time, start_time, units = "secs"))))
+
+ result
}
)
diff --git a/amber/src/main/scala/org/apache/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala b/amber/src/main/scala/org/apache/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala
index b1d6b2f609..871d1a2efb 100644
--- a/amber/src/main/scala/org/apache/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala
+++ b/amber/src/main/scala/org/apache/amber/engine/architecture/worker/promisehandlers/InitializeExecutorHandler.scala
@@ -45,6 +45,7 @@ trait InitializeExecutorHandler {
// Set execution context for this thread
ExecutionContext.setExecutionId(req.executionId)
+ ExecutionContext.setOperatorId(VirtualIdentityUtils.getPhysicalOpId(actorId).logicalOpId.id)
dp.executor = req.opExecInitInfo match {
case OpExecWithClassName(className, descString) =>
diff --git a/common/workflow-core/src/main/scala/org/apache/amber/core/executor/ExecutionContext.scala b/common/workflow-core/src/main/scala/org/apache/amber/core/executor/ExecutionContext.scala
index af33e84d2a..f448f3eb59 100644
--- a/common/workflow-core/src/main/scala/org/apache/amber/core/executor/ExecutionContext.scala
+++ b/common/workflow-core/src/main/scala/org/apache/amber/core/executor/ExecutionContext.scala
@@ -21,10 +21,12 @@ package org.apache.amber.core.executor
/**
* ExecutionContext provides thread-local access to execution metadata.
- * This allows operator executors to access execution ID.
+ * This allows operator executors to access execution ID and operator ID.
*/
object ExecutionContext {
private val executionIdThreadLocal: ThreadLocal[Option[Int]] = ThreadLocal.withInitial(() => None)
+ private val operatorIdThreadLocal: ThreadLocal[Option[String]] =
+ ThreadLocal.withInitial(() => None)
/**
* Sets the execution ID for the current thread.
@@ -43,10 +45,27 @@ object ExecutionContext {
}
/**
- * Clears the execution ID for the current thread.
+ * Sets the operator ID for the current thread.
+ * Should be called when initializing an executor.
+ */
+ def setOperatorId(operatorId: String): Unit = {
+ operatorIdThreadLocal.set(Some(operatorId))
+ }
+
+ /**
+ * Gets the operator ID for the current thread.
+ * @return Some(operatorId) if set, None otherwise
+ */
+ def getOperatorId: Option[String] = {
+ operatorIdThreadLocal.get()
+ }
+
+ /**
+ * Clears the execution context for the current thread.
* Should be called when cleaning up.
*/
def clear(): Unit = {
executionIdThreadLocal.remove()
+ operatorIdThreadLocal.remove()
}
}
diff --git a/common/workflow-operator/src/main/scala/org/apache/amber/operator/source/scan/FileScanSourceOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/amber/operator/source/scan/FileScanSourceOpExec.scala
index 1b5a012b00..5013289c44 100644
--- a/common/workflow-operator/src/main/scala/org/apache/amber/operator/source/scan/FileScanSourceOpExec.scala
+++ b/common/workflow-operator/src/main/scala/org/apache/amber/operator/source/scan/FileScanSourceOpExec.scala
@@ -87,12 +87,16 @@ class FileScanSourceOpExec private[scan] (
new String(toByteArray(entry), desc.fileEncoding.getCharset)
case FileAttributeType.BIG_OBJECT =>
// For big objects, create a big object pointer from the input stream
- // Get execution ID from thread-local context
+ // Get execution ID and operator ID from thread-local context
val executionId = ExecutionContext.getExecutionId
.getOrElse(
throw new IllegalStateException("Execution ID not set in ExecutionContext")
)
- BigObjectManager.create(executionId, entry)
+ val operatorId = ExecutionContext.getOperatorId
+ .getOrElse(
+ throw new IllegalStateException("Operator ID not set in ExecutionContext")
+ )
+ BigObjectManager.create(executionId, operatorId, entry)
case _ => parseField(toByteArray(entry), desc.attributeType.getType)
})
TupleLike(fields.toSeq: _*)
diff --git a/sql/texera_ddl.sql b/sql/texera_ddl.sql
index 188f4b8e98..a7db9ebe15 100644
--- a/sql/texera_ddl.sql
+++ b/sql/texera_ddl.sql
@@ -446,6 +446,7 @@ END $$;
CREATE TABLE big_object (
execution_id INT NOT NULL,
+ operator_id VARCHAR(100) NOT NULL,
uri TEXT NOT NULL UNIQUE,
creation_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (execution_id) REFERENCES workflow_executions(eid) ON DELETE CASCADE
diff --git a/sql/updates/big_object.sql b/sql/updates/big_object.sql
index 0ffaabcc2c..bc762bb346 100644
--- a/sql/updates/big_object.sql
+++ b/sql/updates/big_object.sql
@@ -27,6 +27,7 @@ SET search_path TO texera_db;
-- ============================================
CREATE TABLE big_object (
execution_id INT NOT NULL,
+ operator_id VARCHAR(100) NOT NULL,
uri TEXT NOT NULL UNIQUE,
creation_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (execution_id) REFERENCES workflow_executions(eid) ON DELETE CASCADE
(texera) 03/07: Big Object Read for Python UDF Operators
Posted by ku...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kunwp1 pushed a commit to branch chris-big-object-version-0
in repository https://gitbox.apache.org/repos/asf/texera.git
commit c2c59409e3a332a6eb9c5c9112b92601ec953e92
Author: Kunwoo Park <ku...@uci.edu>
AuthorDate: Wed Oct 15 22:01:16 2025 -0700
Big Object Read for Python UDF Operators
---
amber/operator-requirements.txt | 1 +
.../core/architecture/packaging/output_manager.py | 12 +-
.../python/core/models/schema/attribute_type.py | 6 +
amber/src/main/python/core/models/schema/schema.py | 36 +++--
amber/src/main/python/core/models/tuple.py | 18 ++-
.../main/python/core/storage/big_object_pointer.py | 65 +++++++++
.../main/python/core/storage/document_factory.py | 4 +-
.../python/core/storage/iceberg/iceberg_utils.py | 153 +++++++++++++++++++--
amber/src/main/python/pytexera/__init__.py | 2 +
.../src/main/python/pytexera/big_object_manager.py | 114 +++++++++++++++
common/workflow-core/build.sbt | 4 +-
.../amber/core/tuple/AttributeTypeUtils.scala | 2 +-
.../scala/org/apache/amber/util/ArrowUtils.scala | 62 ++++++---
.../texera/service/util/BigObjectManager.scala | 31 +----
14 files changed, 441 insertions(+), 69 deletions(-)
diff --git a/amber/operator-requirements.txt b/amber/operator-requirements.txt
index fd33ef7470..b9014ba78a 100644
--- a/amber/operator-requirements.txt
+++ b/amber/operator-requirements.txt
@@ -22,3 +22,4 @@ pillow==10.2.0
pybase64==1.3.2
torch==2.8.0
scikit-learn==1.5.0
+boto3==1.40.53
diff --git a/amber/src/main/python/core/architecture/packaging/output_manager.py b/amber/src/main/python/core/architecture/packaging/output_manager.py
index 76be4116b4..eb027920ee 100644
--- a/amber/src/main/python/core/architecture/packaging/output_manager.py
+++ b/amber/src/main/python/core/architecture/packaging/output_manager.py
@@ -259,10 +259,20 @@ class OutputManager:
)
def tuple_to_frame(self, tuples: typing.List[Tuple]) -> DataFrame:
+ from core.storage.big_object_pointer import BigObjectPointer
+
return DataFrame(
frame=Table.from_pydict(
{
- name: [t[name] for t in tuples]
+ name: [
+ (
+ # Convert BigObjectPointer objects to URI strings for Arrow serialization
+ t[name].uri
+ if isinstance(t[name], BigObjectPointer)
+ else t[name]
+ )
+ for t in tuples
+ ]
for name in self.get_port().get_schema().get_attr_names()
},
schema=self.get_port().get_schema().as_arrow_schema(),
diff --git a/amber/src/main/python/core/models/schema/attribute_type.py b/amber/src/main/python/core/models/schema/attribute_type.py
index 72799b015c..63c952d1f8 100644
--- a/amber/src/main/python/core/models/schema/attribute_type.py
+++ b/amber/src/main/python/core/models/schema/attribute_type.py
@@ -20,6 +20,7 @@ import pyarrow as pa
from bidict import bidict
from enum import Enum
from pyarrow import lib
+from core.storage.big_object_pointer import BigObjectPointer
class AttributeType(Enum):
@@ -37,6 +38,7 @@ class AttributeType(Enum):
DOUBLE = 5
TIMESTAMP = 6
BINARY = 7
+ BIG_OBJECT = 8
RAW_TYPE_MAPPING = bidict(
@@ -48,6 +50,7 @@ RAW_TYPE_MAPPING = bidict(
"BOOLEAN": AttributeType.BOOL,
"TIMESTAMP": AttributeType.TIMESTAMP,
"BINARY": AttributeType.BINARY,
+ "BIG_OBJECT": AttributeType.BIG_OBJECT,
}
)
@@ -59,6 +62,7 @@ TO_ARROW_MAPPING = {
AttributeType.BOOL: pa.bool_(),
AttributeType.BINARY: pa.binary(),
AttributeType.TIMESTAMP: pa.timestamp("us"),
+ AttributeType.BIG_OBJECT: pa.string(), # Serialized as URI string
}
FROM_ARROW_MAPPING = {
@@ -83,6 +87,7 @@ TO_PYOBJECT_MAPPING = {
AttributeType.BOOL: bool,
AttributeType.BINARY: bytes,
AttributeType.TIMESTAMP: datetime.datetime,
+ AttributeType.BIG_OBJECT: BigObjectPointer,
}
FROM_PYOBJECT_MAPPING = {
@@ -92,4 +97,5 @@ FROM_PYOBJECT_MAPPING = {
bool: AttributeType.BOOL,
bytes: AttributeType.BINARY,
datetime.datetime: AttributeType.TIMESTAMP,
+ BigObjectPointer: AttributeType.BIG_OBJECT,
}
diff --git a/amber/src/main/python/core/models/schema/schema.py b/amber/src/main/python/core/models/schema/schema.py
index 132ca23884..bcb323f523 100644
--- a/amber/src/main/python/core/models/schema/schema.py
+++ b/amber/src/main/python/core/models/schema/schema.py
@@ -81,26 +81,46 @@ class Schema:
def _from_arrow_schema(self, arrow_schema: pa.Schema) -> None:
"""
Resets the Schema by converting a pyarrow.Schema.
+ Checks field metadata to detect BIG_OBJECT types.
:param arrow_schema: a pyarrow.Schema.
:return:
"""
self._name_type_mapping = OrderedDict()
for attr_name in arrow_schema.names:
- arrow_type = arrow_schema.field(attr_name).type # type: ignore
- attr_type = FROM_ARROW_MAPPING[arrow_type.id]
+ field = arrow_schema.field(attr_name)
+
+ # Check metadata for BIG_OBJECT type (stored by Scala ArrowUtils)
+ is_big_object = (
+ field.metadata and field.metadata.get(b"texera_type") == b"BIG_OBJECT"
+ )
+
+ attr_type = (
+ AttributeType.BIG_OBJECT
+ if is_big_object
+ else FROM_ARROW_MAPPING[field.type.id]
+ )
+
self.add(attr_name, attr_type)
def as_arrow_schema(self) -> pa.Schema:
"""
Creates a new pyarrow.Schema according to the current Schema.
+ Includes metadata for BIG_OBJECT types to preserve type information.
:return: pyarrow.Schema
"""
- return pa.schema(
- [
- pa.field(attr_name, TO_ARROW_MAPPING[attr_type])
- for attr_name, attr_type in self._name_type_mapping.items()
- ]
- )
+ fields = [
+ pa.field(
+ attr_name,
+ TO_ARROW_MAPPING[attr_type],
+ metadata=(
+ {b"texera_type": b"BIG_OBJECT"}
+ if attr_type == AttributeType.BIG_OBJECT
+ else None
+ ),
+ )
+ for attr_name, attr_type in self._name_type_mapping.items()
+ ]
+ return pa.schema(fields)
def get_attr_names(self) -> List[str]:
"""
diff --git a/amber/src/main/python/core/models/tuple.py b/amber/src/main/python/core/models/tuple.py
index e88f08286d..f88d75b4fe 100644
--- a/amber/src/main/python/core/models/tuple.py
+++ b/amber/src/main/python/core/models/tuple.py
@@ -29,6 +29,7 @@ from pympler import asizeof
from typing import Any, List, Iterator, Callable
from typing_extensions import Protocol, runtime_checkable
+from core.storage.big_object_pointer import BigObjectPointer
from .schema.attribute_type import TO_PYOBJECT_MAPPING, AttributeType
from .schema.field import Field
from .schema.schema import Schema
@@ -86,6 +87,7 @@ class ArrowTableTupleProvider:
"""
value = self._table.column(field_name).chunks[chunk_idx][tuple_idx].as_py()
field_type = self._table.schema.field(field_name).type
+ field_metadata = self._table.schema.field(field_name).metadata
# for binary types, convert pickled objects back.
if (
@@ -94,6 +96,16 @@ class ArrowTableTupleProvider:
and value[:6] == b"pickle"
):
value = pickle.loads(value[10:])
+
+ # Convert URI string to BigObjectPointer for BIG_OBJECT types
+ # Metadata is set by Scala ArrowUtils or Python iceberg_utils
+ elif (
+ value is not None
+ and field_metadata
+ and field_metadata.get(b"texera_type") == b"BIG_OBJECT"
+ ):
+ value = BigObjectPointer(value)
+
return value
self._current_idx += 1
@@ -322,8 +334,10 @@ class Tuple:
for field_name, field_value in self.as_key_value_pairs():
expected = schema.get_attr_type(field_name)
- if not isinstance(
- field_value, (TO_PYOBJECT_MAPPING.get(expected), type(None))
+ expected_type = TO_PYOBJECT_MAPPING.get(expected)
+
+ if expected_type is not None and not isinstance(
+ field_value, (expected_type, type(None))
):
raise TypeError(
f"Unmatched type for field '{field_name}', expected {expected}, "
diff --git a/amber/src/main/python/core/storage/big_object_pointer.py b/amber/src/main/python/core/storage/big_object_pointer.py
new file mode 100644
index 0000000000..4aac9d3a63
--- /dev/null
+++ b/amber/src/main/python/core/storage/big_object_pointer.py
@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+BigObjectPointer represents a reference to a large object stored externally (e.g., S3).
+This is a storage reference class used throughout the system for handling big objects.
+"""
+
+
+class BigObjectPointer:
+ """
+ Represents a pointer to a large object stored in S3.
+ The pointer is formatted as a URI: s3://bucket/path/to/object
+
+ This is a lightweight value object that only stores the URI reference.
+ To actually read the object's content, use BigObjectManager.open(pointer).
+ """
+
+ def __init__(self, uri: str):
+ """
+ Create a new BigObjectPointer.
+
+ Args:
+ uri: S3 URI in the format s3://bucket/path/to/object
+
+ Raises:
+ ValueError: If the URI is invalid or doesn't start with s3://
+ """
+ if not uri or not uri.startswith("s3://"):
+ raise ValueError(
+ f"BigObjectPointer URI must start with 's3://' but was: {uri}"
+ )
+ self.uri = uri
+
+ def __str__(self) -> str:
+ """Return the URI as string representation."""
+ return self.uri
+
+ def __repr__(self) -> str:
+ """Return developer-friendly representation."""
+ return f"BigObjectPointer('{self.uri}')"
+
+ def __eq__(self, other) -> bool:
+ """Check equality based on URI."""
+ if isinstance(other, BigObjectPointer):
+ return self.uri == other.uri
+ return False
+
+ def __hash__(self) -> int:
+ """Make BigObjectPointer hashable for use in sets/dicts."""
+ return hash(self.uri)
diff --git a/amber/src/main/python/core/storage/document_factory.py b/amber/src/main/python/core/storage/document_factory.py
index ba15069817..5e680b30fc 100644
--- a/amber/src/main/python/core/storage/document_factory.py
+++ b/amber/src/main/python/core/storage/document_factory.py
@@ -25,6 +25,7 @@ from core.storage.iceberg.iceberg_catalog_instance import IcebergCatalogInstance
from core.storage.iceberg.iceberg_document import IcebergDocument
from core.storage.iceberg.iceberg_utils import (
create_table,
+ amber_schema_to_iceberg_schema,
amber_tuples_to_arrow_table,
arrow_table_to_amber_tuples,
load_table_metadata,
@@ -63,7 +64,8 @@ class DocumentFactory:
if resource_type in {VFSResourceType.RESULT}:
storage_key = DocumentFactory.sanitize_uri_path(parsed_uri)
- iceberg_schema = Schema.as_arrow_schema(schema)
+ # Convert Amber Schema to Iceberg Schema with BIG_OBJECT field name encoding
+ iceberg_schema = amber_schema_to_iceberg_schema(schema)
create_table(
IcebergCatalogInstance.get_instance(),
diff --git a/amber/src/main/python/core/storage/iceberg/iceberg_utils.py b/amber/src/main/python/core/storage/iceberg/iceberg_utils.py
index 12c841ed71..3c23df73b0 100644
--- a/amber/src/main/python/core/storage/iceberg/iceberg_utils.py
+++ b/amber/src/main/python/core/storage/iceberg/iceberg_utils.py
@@ -29,6 +29,112 @@ from typing import Optional, Iterable
import core
from core.models import ArrowTableTupleProvider, Tuple
+# Suffix used to encode BIG_OBJECT fields in Iceberg (must match Scala IcebergUtil)
+BIG_OBJECT_FIELD_SUFFIX = "__texera_big_obj_ptr"
+
+# Shared type mappings
+_ICEBERG_TO_AMBER_TYPE_MAPPING = {
+ "string": "STRING",
+ "int": "INT",
+ "integer": "INT",
+ "long": "LONG",
+ "double": "DOUBLE",
+ "float": "DOUBLE",
+ "boolean": "BOOL",
+ "timestamp": "TIMESTAMP",
+ "binary": "BINARY",
+}
+
+
+def encode_big_object_field_name(field_name: str, attr_type) -> str:
+ """Encodes BIG_OBJECT field names with suffix for Iceberg storage."""
+ from core.models.schema.attribute_type import AttributeType
+
+ return (
+ f"{field_name}{BIG_OBJECT_FIELD_SUFFIX}"
+ if attr_type == AttributeType.BIG_OBJECT
+ else field_name
+ )
+
+
+def decode_big_object_field_name(field_name: str) -> str:
+ """Decodes field names by removing BIG_OBJECT suffix if present."""
+ return (
+ field_name[: -len(BIG_OBJECT_FIELD_SUFFIX)]
+ if field_name.endswith(BIG_OBJECT_FIELD_SUFFIX)
+ else field_name
+ )
+
+
+def iceberg_schema_to_amber_schema(iceberg_schema: Schema):
+ """
+ Converts PyIceberg Schema to Amber Schema.
+ Decodes BIG_OBJECT field names and adds Arrow metadata.
+ """
+ from core.models.schema.attribute_type import AttributeType, TO_ARROW_MAPPING
+ import core.models
+
+ arrow_fields = []
+ for field in iceberg_schema.fields:
+ # Decode field name and detect BIG_OBJECT by suffix
+ decoded_name = decode_big_object_field_name(field.name)
+ is_big_object = field.name.endswith(BIG_OBJECT_FIELD_SUFFIX)
+
+ # Determine attribute type
+ if is_big_object:
+ attr_type = AttributeType.BIG_OBJECT
+ else:
+ iceberg_type_str = str(field.field_type).lower()
+ attr_type_name = _ICEBERG_TO_AMBER_TYPE_MAPPING.get(
+ iceberg_type_str, "STRING"
+ )
+ attr_type = getattr(AttributeType, attr_type_name)
+
+ # Create Arrow field with metadata
+ arrow_field = pa.field(
+ decoded_name,
+ TO_ARROW_MAPPING[attr_type],
+ metadata={b"texera_type": b"BIG_OBJECT"} if is_big_object else None,
+ )
+ arrow_fields.append(arrow_field)
+
+ return core.models.Schema(pa.schema(arrow_fields))
+
+
+def amber_schema_to_iceberg_schema(amber_schema) -> Schema:
+ """
+ Converts Amber Schema to PyIceberg Schema.
+ Encodes BIG_OBJECT field names with suffix.
+ """
+ from pyiceberg import types as iceberg_types
+ from core.models.schema.attribute_type import AttributeType
+
+ # Mapping from Amber AttributeType to Iceberg Type
+ TYPE_MAPPING = {
+ AttributeType.STRING: iceberg_types.StringType(),
+ AttributeType.INT: iceberg_types.IntegerType(),
+ AttributeType.LONG: iceberg_types.LongType(),
+ AttributeType.DOUBLE: iceberg_types.DoubleType(),
+ AttributeType.BOOL: iceberg_types.BooleanType(),
+ AttributeType.TIMESTAMP: iceberg_types.TimestampType(),
+ AttributeType.BINARY: iceberg_types.BinaryType(),
+ AttributeType.BIG_OBJECT: iceberg_types.StringType(),
+ }
+
+ fields = [
+ iceberg_types.NestedField(
+ field_id=idx,
+ name=encode_big_object_field_name(field_name, attr_type),
+ field_type=TYPE_MAPPING[attr_type],
+ required=False,
+ )
+ for idx, (field_name, attr_type) in enumerate(
+ amber_schema._name_type_mapping.items(), start=1
+ )
+ ]
+
+ return Schema(*fields)
+
def create_postgres_catalog(
catalog_name: str,
@@ -135,12 +241,29 @@ def amber_tuples_to_arrow_table(
) -> pa.Table:
"""
Converts a list of amber tuples to a pyarrow table for serialization.
+ Handles BIG_OBJECT field name encoding and serialization.
"""
+ from core.storage.big_object_pointer import BigObjectPointer
+
+ # Build data dict using Iceberg schema field names (encoded)
+ data_dict = {}
+ for encoded_name in iceberg_schema.as_arrow().names:
+ # Decode to get the original field name used in tuples
+ decoded_name = decode_big_object_field_name(encoded_name)
+
+ # Extract values from tuples and serialize BigObjectPointer
+ values = []
+ for t in tuple_list:
+ value = t[decoded_name]
+ # Serialize BigObjectPointer to URI string for Iceberg storage
+ if isinstance(value, BigObjectPointer):
+ value = value.uri
+ values.append(value)
+
+ data_dict[encoded_name] = values
+
return pa.Table.from_pydict(
- {
- name: [t[name] for t in tuple_list]
- for name in iceberg_schema.as_arrow().names
- },
+ data_dict,
schema=iceberg_schema.as_arrow(),
)
@@ -149,13 +272,27 @@ def arrow_table_to_amber_tuples(
iceberg_schema: Schema, arrow_table: pa.Table
) -> Iterable[Tuple]:
"""
- Converts an arrow table to a list of amber tuples for deserialization.
+ Converts an arrow table read from Iceberg to Amber tuples.
+ Properly handles BIG_OBJECT field name decoding and type detection.
"""
- tuple_provider = ArrowTableTupleProvider(arrow_table)
+ # Convert Iceberg schema to Amber schema with proper metadata
+ amber_schema = iceberg_schema_to_amber_schema(iceberg_schema)
+
+ # Create an Arrow table with proper metadata by rebuilding it with the correct schema
+ # This ensures BIG_OBJECT metadata is available during tuple field access
+ arrow_table_with_metadata = pa.Table.from_arrays(
+ [arrow_table.column(col_name) for col_name in arrow_table.column_names],
+ schema=amber_schema.as_arrow_schema(),
+ )
+
+ tuple_provider = ArrowTableTupleProvider(arrow_table_with_metadata)
return (
Tuple(
- {name: field_accessor for name in arrow_table.column_names},
- schema=core.models.Schema(iceberg_schema.as_arrow()),
+ {
+ decode_big_object_field_name(name): field_accessor
+ for name in arrow_table.column_names
+ },
+ schema=amber_schema,
)
for field_accessor in tuple_provider
)
diff --git a/amber/src/main/python/pytexera/__init__.py b/amber/src/main/python/pytexera/__init__.py
index db78319f49..2b3d59be10 100644
--- a/amber/src/main/python/pytexera/__init__.py
+++ b/amber/src/main/python/pytexera/__init__.py
@@ -27,6 +27,7 @@ from .udf.udf_operator import (
UDFBatchOperator,
UDFSourceOperator,
)
+from .big_object_manager import BigObjectManager
__all__ = [
"State",
@@ -41,6 +42,7 @@ __all__ = [
"UDFBatchOperator",
"UDFSourceOperator",
"DatasetFileDocument",
+ "BigObjectManager",
# export external tools to be used
"overrides",
"logger",
diff --git a/amber/src/main/python/pytexera/big_object_manager.py b/amber/src/main/python/pytexera/big_object_manager.py
new file mode 100644
index 0000000000..f61549fb5f
--- /dev/null
+++ b/amber/src/main/python/pytexera/big_object_manager.py
@@ -0,0 +1,114 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""BigObjectManager for reading large objects from S3."""
+
+import os
+from typing import BinaryIO
+from core.storage.big_object_pointer import BigObjectPointer
+
+
+class BigObjectStream:
+ """Stream for reading big objects (matches Scala BigObjectStream)."""
+
+ def __init__(self, body: BinaryIO, pointer: BigObjectPointer):
+ self._body = body
+ self._pointer = pointer
+ self._closed = False
+
+ def read(self, amt=None):
+ if self._closed:
+ raise ValueError("I/O operation on closed stream")
+ return self._body.read(amt)
+
+ def close(self):
+ if not self._closed:
+ self._closed = True
+ self._body.close()
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.close()
+
+ @property
+ def closed(self):
+ return self._closed
+
+
+class BigObjectManager:
+ """Manager for reading big objects from S3."""
+
+ _s3_client = None
+
+ @classmethod
+ def _get_s3_client(cls):
+ """Initialize S3 client (lazy, cached)."""
+ if cls._s3_client is None:
+ try:
+ import boto3
+ from botocore.config import Config
+
+ cls._s3_client = boto3.client(
+ "s3",
+ endpoint_url=os.environ.get(
+ "STORAGE_S3_ENDPOINT", "http://localhost:9000"
+ ),
+ aws_access_key_id=os.environ.get(
+ "STORAGE_S3_AUTH_USERNAME", "texera_minio"
+ ),
+ aws_secret_access_key=os.environ.get(
+ "STORAGE_S3_AUTH_PASSWORD", "password"
+ ),
+ region_name=os.environ.get("STORAGE_S3_REGION", "us-west-2"),
+ config=Config(
+ signature_version="s3v4", s3={"addressing_style": "path"}
+ ),
+ )
+ except ImportError:
+ raise RuntimeError("boto3 required. Install with: pip install boto3")
+ except Exception as e:
+ raise RuntimeError(f"Failed to initialize S3 client: {e}")
+
+ return cls._s3_client
+
+ @classmethod
+ def open(cls, pointer: BigObjectPointer) -> BigObjectStream:
+ """
+ Open a big object for reading.
+
+ Usage: with BigObjectManager.open(tuple_["document"]) as stream:
+ content = stream.read().decode('utf-8')
+ """
+ if not isinstance(pointer, BigObjectPointer):
+ raise TypeError(f"Expected BigObjectPointer, got {type(pointer)}")
+
+ # Parse s3://bucket/key
+ parts = pointer.uri.removeprefix("s3://").split("/", 1)
+ if len(parts) != 2:
+ raise ValueError(
+ f"Invalid S3 URI (expected s3://bucket/key): {pointer.uri}"
+ )
+
+ bucket, key = parts
+
+ try:
+ body = cls._get_s3_client().get_object(Bucket=bucket, Key=key)["Body"]
+ return BigObjectStream(body, pointer)
+ except Exception as e:
+ raise RuntimeError(f"Failed to open {pointer.uri}: {e}")
diff --git a/common/workflow-core/build.sbt b/common/workflow-core/build.sbt
index 122c267271..e4cd11c197 100644
--- a/common/workflow-core/build.sbt
+++ b/common/workflow-core/build.sbt
@@ -183,7 +183,9 @@ libraryDependencies ++= Seq(
"io.lakefs" % "sdk" % "1.51.0", // for lakeFS api calls
"com.typesafe" % "config" % "1.4.3", // config reader
"org.apache.commons" % "commons-jcs3-core" % "3.2", // Apache Commons JCS
- "software.amazon.awssdk" % "s3" % "2.29.51",
+ "software.amazon.awssdk" % "s3" % "2.29.51" excludeAll(
+ ExclusionRule(organization = "io.netty")
+ ),
"software.amazon.awssdk" % "auth" % "2.29.51",
"software.amazon.awssdk" % "regions" % "2.29.51",
)
\ No newline at end of file
diff --git a/common/workflow-core/src/main/scala/org/apache/amber/core/tuple/AttributeTypeUtils.scala b/common/workflow-core/src/main/scala/org/apache/amber/core/tuple/AttributeTypeUtils.scala
index 8184560204..0efeea960f 100644
--- a/common/workflow-core/src/main/scala/org/apache/amber/core/tuple/AttributeTypeUtils.scala
+++ b/common/workflow-core/src/main/scala/org/apache/amber/core/tuple/AttributeTypeUtils.scala
@@ -128,7 +128,7 @@ object AttributeTypeUtils extends Serializable {
case AttributeType.TIMESTAMP => parseTimestamp(field)
case AttributeType.STRING => field.toString
case AttributeType.BINARY => field
- case AttributeType.BIG_OBJECT => field // Big objects are created programmatically, not parsed
+ case AttributeType.BIG_OBJECT => new BigObjectPointer(field.toString)
case AttributeType.ANY | _ => field
}
}
diff --git a/common/workflow-core/src/main/scala/org/apache/amber/util/ArrowUtils.scala b/common/workflow-core/src/main/scala/org/apache/amber/util/ArrowUtils.scala
index 25443d5dc7..20f8775974 100644
--- a/common/workflow-core/src/main/scala/org/apache/amber/util/ArrowUtils.scala
+++ b/common/workflow-core/src/main/scala/org/apache/amber/util/ArrowUtils.scala
@@ -73,28 +73,28 @@ object ArrowUtils extends LazyLogging {
Tuple
.builder(schema)
.addSequentially(
- vectorSchemaRoot.getFieldVectors.asScala
- .map((fieldVector: FieldVector) => {
+ vectorSchemaRoot.getFieldVectors.asScala.zipWithIndex.map {
+ case (fieldVector: FieldVector, index: Int) =>
val value: AnyRef = fieldVector.getObject(rowIndex)
try {
- val arrowType = fieldVector.getField.getFieldType.getType
- val attributeType = toAttributeType(arrowType)
+ // Use the attribute type from the schema (which includes metadata)
+ // instead of deriving it from the Arrow type
+ val attributeType = schema.getAttributes(index).getType
AttributeTypeUtils.parseField(value, attributeType)
-
} catch {
case e: Exception =>
logger.warn("Caught error during parsing Arrow value back to Texera value", e)
null
}
- })
- .toArray
+ }.toArray
)
.build()
}
/**
* Converts an Arrow Schema into Texera Schema.
+ * Checks field metadata to detect BIG_OBJECT types.
*
* @param arrowSchema The Arrow Schema to be converted.
* @return A Texera Schema.
@@ -102,7 +102,12 @@ object ArrowUtils extends LazyLogging {
def toTexeraSchema(arrowSchema: org.apache.arrow.vector.types.pojo.Schema): Schema =
Schema(
arrowSchema.getFields.asScala.map { field =>
- new Attribute(field.getName, toAttributeType(field.getType))
+ val isBigObject = Option(field.getMetadata)
+ .exists(m => m.containsKey("texera_type") && m.get("texera_type") == "BIG_OBJECT")
+
+ val attributeType =
+ if (isBigObject) AttributeType.BIG_OBJECT else toAttributeType(field.getType)
+ new Attribute(field.getName, attributeType)
}.toList
)
@@ -207,11 +212,17 @@ object ArrowUtils extends LazyLogging {
)
case _: ArrowType.Utf8 =>
- if (isNull) vector.asInstanceOf[VarCharVector].setNull(index)
- else
+ if (isNull) {
+ vector.asInstanceOf[VarCharVector].setNull(index)
+ } else {
+ val stringValue = value match {
+ case ptr: BigObjectPointer => ptr.getUri
+ case _ => value.toString
+ }
vector
.asInstanceOf[VarCharVector]
- .setSafe(index, value.asInstanceOf[String].getBytes(StandardCharsets.UTF_8))
+ .setSafe(index, stringValue.getBytes(StandardCharsets.UTF_8))
+ }
case _: ArrowType.Binary | _: ArrowType.LargeBinary =>
if (isNull) vector.asInstanceOf[VarBinaryVector].setNull(index)
else
@@ -227,19 +238,32 @@ object ArrowUtils extends LazyLogging {
/**
* Converts an Amber schema into Arrow schema.
+ * Stores AttributeType in field metadata to preserve BIG_OBJECT type information.
*
* @param schema The Texera Schema.
* @return An Arrow Schema.
*/
def fromTexeraSchema(schema: Schema): org.apache.arrow.vector.types.pojo.Schema = {
- val arrowFields = new util.ArrayList[Field]
-
- for (amberAttribute <- schema.getAttributes) {
- val name = amberAttribute.getName
- val field = Field.nullablePrimitive(name, fromAttributeType(amberAttribute.getType))
- arrowFields.add(field)
+ val arrowFields = schema.getAttributes.map { attribute =>
+ val metadata = if (attribute.getType == AttributeType.BIG_OBJECT) {
+ val map = new util.HashMap[String, String]()
+ map.put("texera_type", "BIG_OBJECT")
+ map
+ } else null
+
+ new Field(
+ attribute.getName,
+ new org.apache.arrow.vector.types.pojo.FieldType(
+ true, // nullable
+ fromAttributeType(attribute.getType),
+ null, // dictionary encoding
+ metadata
+ ),
+ null // children
+ )
}
- new org.apache.arrow.vector.types.pojo.Schema(arrowFields)
+
+ new org.apache.arrow.vector.types.pojo.Schema(util.Arrays.asList(arrowFields: _*))
}
/**
@@ -270,7 +294,7 @@ object ArrowUtils extends LazyLogging {
case AttributeType.BINARY =>
new ArrowType.Binary
- case AttributeType.STRING | AttributeType.ANY =>
+ case AttributeType.STRING | AttributeType.BIG_OBJECT | AttributeType.ANY =>
ArrowType.Utf8.INSTANCE
case _ =>
diff --git a/common/workflow-core/src/main/scala/org/apache/texera/service/util/BigObjectManager.scala b/common/workflow-core/src/main/scala/org/apache/texera/service/util/BigObjectManager.scala
index 19586372cc..86688fd6b7 100644
--- a/common/workflow-core/src/main/scala/org/apache/texera/service/util/BigObjectManager.scala
+++ b/common/workflow-core/src/main/scala/org/apache/texera/service/util/BigObjectManager.scala
@@ -26,15 +26,12 @@ import org.apache.texera.dao.jooq.generated.Tables.BIG_OBJECT
import java.io.{Closeable, InputStream}
import java.util.UUID
-import java.util.concurrent.ConcurrentHashMap
import scala.jdk.CollectionConverters._
/**
- * BigObjectStream wraps an InputStream and tracks its lifecycle for proper cleanup.
+ * BigObjectStream wraps an InputStream for reading big objects.
*/
-class BigObjectStream(private val inputStream: InputStream, private val pointer: BigObjectPointer)
- extends InputStream
- with Closeable {
+class BigObjectStream(private val inputStream: InputStream) extends InputStream with Closeable {
@volatile private var closed = false
@@ -63,12 +60,10 @@ class BigObjectStream(private val inputStream: InputStream, private val pointer:
if (!closed) {
closed = true
inputStream.close()
- BigObjectManager.closeStream(pointer)
}
}
def isClosed: Boolean = closed
- def getPointer: BigObjectPointer = pointer
}
/**
@@ -76,7 +71,6 @@ class BigObjectStream(private val inputStream: InputStream, private val pointer:
*/
object BigObjectManager extends LazyLogging {
private val DEFAULT_BUCKET = "texera-big-objects"
- private val openStreams = new ConcurrentHashMap[BigObjectPointer, BigObjectStream]()
private lazy val context = SqlServer.getInstance().createDSLContext()
/**
@@ -134,9 +128,7 @@ object BigObjectManager extends LazyLogging {
)
val inputStream = S3StorageClient.downloadObject(ptr.getBucketName, ptr.getObjectKey)
- val stream = new BigObjectStream(inputStream, ptr)
- openStreams.put(ptr, stream)
- stream
+ new BigObjectStream(inputStream)
}
/**
@@ -166,7 +158,6 @@ object BigObjectManager extends LazyLogging {
uris.foreach { uri =>
try {
val ptr = new BigObjectPointer(uri)
- Option(openStreams.get(ptr)).foreach(_.close())
S3StorageClient.deleteObject(ptr.getBucketName, ptr.getObjectKey)
} catch {
case e: Exception =>
@@ -180,20 +171,4 @@ object BigObjectManager extends LazyLogging {
.where(BIG_OBJECT.EXECUTION_ID.eq(executionId))
.execute()
}
-
- /**
- * Closes a big object stream. Typically called automatically when the stream is closed.
- */
- def close(ptr: BigObjectPointer): Unit = {
- Option(openStreams.get(ptr)).foreach { stream =>
- if (!stream.isClosed) stream.close()
- }
- }
-
- /**
- * Internal method to remove a stream from tracking when it's closed.
- */
- private[util] def closeStream(ptr: BigObjectPointer): Unit = {
- openStreams.remove(ptr)
- }
}
(texera) 06/07: create() function for python udf
Posted by ku...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kunwp1 pushed a commit to branch chris-big-object-version-0
in repository https://gitbox.apache.org/repos/asf/texera.git
commit baa5504e1bd324def2b8d97a87b55d7c7ac7600c
Author: Kunwoo Park <ku...@uci.edu>
AuthorDate: Mon Oct 27 10:59:30 2025 -0700
create() function for python udf
---
.../src/main/python/core/storage/storage_config.py | 34 ++++-
.../python/pytexera/storage/big_object_manager.py | 151 +++++++++++++++++++--
amber/src/main/python/texera_run_python_worker.py | 14 ++
.../pythonworker/PythonWorkflowWorker.scala | 9 +-
.../texera/service/util/BigObjectManager.scala | 9 +-
5 files changed, 199 insertions(+), 18 deletions(-)
diff --git a/amber/src/main/python/core/storage/storage_config.py b/amber/src/main/python/core/storage/storage_config.py
index 21c918d514..33ea9654ec 100644
--- a/amber/src/main/python/core/storage/storage_config.py
+++ b/amber/src/main/python/core/storage/storage_config.py
@@ -25,6 +25,7 @@ class StorageConfig:
_initialized = False
+ # Iceberg configs
ICEBERG_POSTGRES_CATALOG_URI_WITHOUT_SCHEME = None
ICEBERG_POSTGRES_CATALOG_USERNAME = None
ICEBERG_POSTGRES_CATALOG_PASSWORD = None
@@ -32,6 +33,17 @@ class StorageConfig:
ICEBERG_FILE_STORAGE_DIRECTORY_PATH = None
ICEBERG_TABLE_COMMIT_BATCH_SIZE = None
+ # S3 configs (for BigObjectManager)
+ S3_ENDPOINT = None
+ S3_REGION = None
+ S3_AUTH_USERNAME = None
+ S3_AUTH_PASSWORD = None
+
+ # JDBC configs (for BigObjectManager database registration)
+ JDBC_URL = None
+ JDBC_USERNAME = None
+ JDBC_PASSWORD = None
+
@classmethod
def initialize(
cls,
@@ -41,18 +53,38 @@ class StorageConfig:
table_result_namespace,
directory_path,
commit_batch_size,
+ s3_endpoint,
+ s3_region,
+ s3_auth_username,
+ s3_auth_password,
+ jdbc_url,
+ jdbc_username,
+ jdbc_password,
):
if cls._initialized:
raise RuntimeError(
- "Storage config has already been initialized" "and cannot be modified."
+ "Storage config has already been initialized and cannot be modified."
)
+ # Iceberg configs
cls.ICEBERG_POSTGRES_CATALOG_URI_WITHOUT_SCHEME = postgres_uri_without_scheme
cls.ICEBERG_POSTGRES_CATALOG_USERNAME = postgres_username
cls.ICEBERG_POSTGRES_CATALOG_PASSWORD = postgres_password
cls.ICEBERG_TABLE_RESULT_NAMESPACE = table_result_namespace
cls.ICEBERG_FILE_STORAGE_DIRECTORY_PATH = directory_path
cls.ICEBERG_TABLE_COMMIT_BATCH_SIZE = int(commit_batch_size)
+
+ # S3 configs
+ cls.S3_ENDPOINT = s3_endpoint
+ cls.S3_REGION = s3_region
+ cls.S3_AUTH_USERNAME = s3_auth_username
+ cls.S3_AUTH_PASSWORD = s3_auth_password
+
+ # JDBC configs
+ cls.JDBC_URL = jdbc_url
+ cls.JDBC_USERNAME = jdbc_username
+ cls.JDBC_PASSWORD = jdbc_password
+
cls._initialized = True
def __new__(cls, *args, **kwargs):
diff --git a/amber/src/main/python/pytexera/storage/big_object_manager.py b/amber/src/main/python/pytexera/storage/big_object_manager.py
index a59129c2c9..c116c1e392 100644
--- a/amber/src/main/python/pytexera/storage/big_object_manager.py
+++ b/amber/src/main/python/pytexera/storage/big_object_manager.py
@@ -17,9 +17,12 @@
"""BigObjectManager for reading large objects from S3."""
-import os
-from typing import BinaryIO
+import time
+import uuid
+from typing import BinaryIO, Union
+from io import BytesIO
from core.models.schema.big_object_pointer import BigObjectPointer
+from core.storage.storage_config import StorageConfig
class BigObjectStream:
@@ -58,6 +61,8 @@ class BigObjectManager:
"""Manager for reading big objects from S3."""
_s3_client = None
+ _db_connection = None
+ DEFAULT_BUCKET = "texera-big-objects"
@classmethod
def _get_s3_client(cls):
@@ -69,16 +74,10 @@ class BigObjectManager:
cls._s3_client = boto3.client(
"s3",
- endpoint_url=os.environ.get(
- "STORAGE_S3_ENDPOINT", "http://localhost:9000"
- ),
- aws_access_key_id=os.environ.get(
- "STORAGE_S3_AUTH_USERNAME", "texera_minio"
- ),
- aws_secret_access_key=os.environ.get(
- "STORAGE_S3_AUTH_PASSWORD", "password"
- ),
- region_name=os.environ.get("STORAGE_S3_REGION", "us-west-2"),
+ endpoint_url=StorageConfig.S3_ENDPOINT,
+ aws_access_key_id=StorageConfig.S3_AUTH_USERNAME,
+ aws_secret_access_key=StorageConfig.S3_AUTH_PASSWORD,
+ region_name=StorageConfig.S3_REGION,
config=Config(
signature_version="s3v4", s3={"addressing_style": "path"}
),
@@ -90,6 +89,134 @@ class BigObjectManager:
return cls._s3_client
+ @classmethod
+ def _get_db_connection(cls):
+ """Get database connection for registering big objects."""
+ if cls._db_connection is None:
+ try:
+ import psycopg2
+ # Parse JDBC URL to extract host, port, database
+ # Format: jdbc:postgresql://host:port/database?params
+ jdbc_url = StorageConfig.JDBC_URL
+ if jdbc_url.startswith("jdbc:postgresql://"):
+ url_part = jdbc_url.replace("jdbc:postgresql://", "")
+ # Split by '?' to separate params
+ url_without_params = url_part.split("?")[0]
+ # Split by '/' to get host:port and database
+ parts = url_without_params.split("/")
+ host_port = parts[0]
+ database = parts[1] if len(parts) > 1 else "texera_db"
+
+ # Split host and port
+ if ":" in host_port:
+ host, port = host_port.split(":")
+ port = int(port)
+ else:
+ host = host_port
+ port = 5432
+
+ cls._db_connection = psycopg2.connect(
+ host=host,
+ port=port,
+ user=StorageConfig.JDBC_USERNAME,
+ password=StorageConfig.JDBC_PASSWORD,
+ database=database
+ )
+ else:
+ raise ValueError(f"Invalid JDBC URL format: {jdbc_url}")
+ except ImportError:
+ raise RuntimeError("psycopg2 required. Install with: pip install psycopg2-binary")
+ except Exception as e:
+ raise RuntimeError(f"Failed to connect to database: {e}")
+ return cls._db_connection
+
+ @classmethod
+ def _create_bucket_if_not_exist(cls, bucket: str):
+ """Create S3 bucket if it doesn't exist."""
+ s3 = cls._get_s3_client()
+ try:
+ s3.head_bucket(Bucket=bucket)
+ except:
+ # Bucket doesn't exist, create it
+ try:
+ s3.create_bucket(Bucket=bucket)
+ print(f"Created S3 bucket: {bucket}")
+ except Exception as e:
+ raise RuntimeError(f"Failed to create bucket {bucket}: {e}")
+
+ @classmethod
+ def create(cls, execution_id: int, operator_id: str, data: Union[bytes, BinaryIO]) -> BigObjectPointer:
+ """
+ Creates a big object from bytes or a stream and uploads to S3.
+ Registers the big object in the database for cleanup tracking.
+
+ Args:
+ execution_id: The execution ID this big object belongs to.
+ operator_id: The operator ID that created this big object.
+ data: Either bytes or a file-like object (stream) containing the data.
+
+ Returns:
+ A BigObjectPointer that can be used in tuples.
+
+ Usage:
+ # From bytes
+ pointer = BigObjectManager.create(execution_id, operator_id, b"large data...")
+
+ # From file
+ with open("large_file.bin", "rb") as f:
+ pointer = BigObjectManager.create(execution_id, operator_id, f)
+
+ # From BytesIO
+ buffer = BytesIO(b"data...")
+ pointer = BigObjectManager.create(execution_id, operator_id, buffer)
+ """
+ cls._create_bucket_if_not_exist(cls.DEFAULT_BUCKET)
+
+ # Generate unique object key
+ timestamp = int(time.time() * 1000) # milliseconds
+ unique_id = str(uuid.uuid4())
+ object_key = f"{timestamp}/{unique_id}"
+ uri = f"s3://{cls.DEFAULT_BUCKET}/{object_key}"
+
+ s3 = cls._get_s3_client()
+
+ try:
+ # Upload to S3
+ if isinstance(data, bytes):
+ # Upload bytes directly
+ s3.put_object(Bucket=cls.DEFAULT_BUCKET, Key=object_key, Body=data)
+ else:
+ # Upload from stream (file-like object)
+ s3.upload_fileobj(data, cls.DEFAULT_BUCKET, object_key)
+
+ print(f"Uploaded big object to S3: {uri}")
+
+ except Exception as e:
+ raise RuntimeError(f"Failed to upload big object to S3: {e}")
+
+ # Register in database
+ try:
+ conn = cls._get_db_connection()
+ cursor = conn.cursor()
+ cursor.execute(
+ "INSERT INTO big_object (execution_id, operator_id, uri) VALUES (%s, %s, %s)",
+ (execution_id, operator_id, uri)
+ )
+ conn.commit()
+ cursor.close()
+ print(f"Registered big object in database: eid={execution_id}, opid={operator_id}, uri={uri}")
+
+ except Exception as e:
+ # Database registration failed - clean up S3 object
+ print(f"Failed to register big object in database, cleaning up: {uri}")
+ try:
+ s3.delete_object(Bucket=cls.DEFAULT_BUCKET, Key=object_key)
+ except Exception as cleanup_error:
+ print(f"Failed to cleanup orphaned S3 object {uri}: {cleanup_error}")
+ raise RuntimeError(f"Failed to create big object: {e}")
+
+ return BigObjectPointer(uri)
+
@classmethod
def open(cls, pointer: BigObjectPointer) -> BigObjectStream:
"""
diff --git a/amber/src/main/python/texera_run_python_worker.py b/amber/src/main/python/texera_run_python_worker.py
index c9594cc218..b3a8f8e341 100644
--- a/amber/src/main/python/texera_run_python_worker.py
+++ b/amber/src/main/python/texera_run_python_worker.py
@@ -51,6 +51,13 @@ if __name__ == "__main__":
iceberg_table_namespace,
iceberg_file_storage_directory_path,
iceberg_table_commit_batch_size,
+ s3_endpoint,
+ s3_region,
+ s3_auth_username,
+ s3_auth_password,
+ jdbc_url,
+ jdbc_username,
+ jdbc_password,
) = sys.argv
init_loguru_logger(logger_level)
StorageConfig.initialize(
@@ -60,6 +67,13 @@ if __name__ == "__main__":
iceberg_table_namespace,
iceberg_file_storage_directory_path,
iceberg_table_commit_batch_size,
+ s3_endpoint,
+ s3_region,
+ s3_auth_username,
+ s3_auth_password,
+ jdbc_url,
+ jdbc_username,
+ jdbc_password,
)
# Setting R_HOME environment variable for R-UDF usage
diff --git a/amber/src/main/scala/org/apache/amber/engine/architecture/pythonworker/PythonWorkflowWorker.scala b/amber/src/main/scala/org/apache/amber/engine/architecture/pythonworker/PythonWorkflowWorker.scala
index fa765aee1f..d9469bf6ba 100644
--- a/amber/src/main/scala/org/apache/amber/engine/architecture/pythonworker/PythonWorkflowWorker.scala
+++ b/amber/src/main/scala/org/apache/amber/engine/architecture/pythonworker/PythonWorkflowWorker.scala
@@ -183,7 +183,14 @@ class PythonWorkflowWorker(
StorageConfig.icebergPostgresCatalogPassword,
StorageConfig.icebergTableResultNamespace,
StorageConfig.fileStorageDirectoryPath.toString,
- StorageConfig.icebergTableCommitBatchSize.toString
+ StorageConfig.icebergTableCommitBatchSize.toString,
+ StorageConfig.s3Endpoint,
+ StorageConfig.s3Region,
+ StorageConfig.s3Username,
+ StorageConfig.s3Password,
+ StorageConfig.jdbcUrl,
+ StorageConfig.jdbcUsername,
+ StorageConfig.jdbcPassword
)
).run(BasicIO.standard(false))
}
diff --git a/common/workflow-core/src/main/scala/org/apache/texera/service/util/BigObjectManager.scala b/common/workflow-core/src/main/scala/org/apache/texera/service/util/BigObjectManager.scala
index 86688fd6b7..6fcdb7b0e5 100644
--- a/common/workflow-core/src/main/scala/org/apache/texera/service/util/BigObjectManager.scala
+++ b/common/workflow-core/src/main/scala/org/apache/texera/service/util/BigObjectManager.scala
@@ -79,10 +79,11 @@ object BigObjectManager extends LazyLogging {
* Registers the big object in the database for cleanup tracking.
*
* @param executionId The execution ID this big object belongs to.
+ * @param operatorId The operator ID that created this big object.
* @param stream The input stream containing the big object data.
* @return A BigObjectPointer that can be used in tuples.
*/
- def create(executionId: Int, stream: InputStream): BigObjectPointer = {
+ def create(executionId: Int, operatorId: String, stream: InputStream): BigObjectPointer = {
S3StorageClient.createBucketIfNotExist(DEFAULT_BUCKET)
val objectKey = s"${System.currentTimeMillis()}/${UUID.randomUUID()}"
@@ -95,10 +96,10 @@ object BigObjectManager extends LazyLogging {
try {
context
.insertInto(BIG_OBJECT)
- .columns(BIG_OBJECT.EXECUTION_ID, BIG_OBJECT.URI)
- .values(Int.box(executionId), uri)
+ .columns(BIG_OBJECT.EXECUTION_ID, BIG_OBJECT.OPERATOR_ID, BIG_OBJECT.URI)
+ .values(Int.box(executionId), operatorId, uri)
.execute()
- logger.debug(s"Registered big object: eid=$executionId, uri=$uri")
+ logger.debug(s"Registered big object: eid=$executionId, opid=$operatorId, uri=$uri")
} catch {
case e: Exception =>
// Database failed - clean up S3 object
(texera) 04/07: Big Object Read for R UDF Operators
Posted by ku...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kunwp1 pushed a commit to branch chris-big-object-version-0
in repository https://gitbox.apache.org/repos/asf/texera.git
commit 7d98b74e727d8b30efa16afa4257f01f2a34c4eb
Author: Kunwoo Park <ku...@uci.edu>
AuthorDate: Fri Oct 17 12:11:51 2025 -0700
Big Object Read for R UDF Operators
---
.../core/architecture/packaging/output_manager.py | 2 +-
.../src/main/python/core/models/RTableExecutor.py | 108 +++++++++++++++++++--
.../src/main/python/core/models/RTupleExecutor.py | 30 +++++-
.../main/python/core/models/big_object_manager.R | 99 +++++++++++++++++++
.../main/python/core/models/big_object_pointer.R | 51 ++++++++++
amber/src/main/python/core/models/r_utils.py | 18 ++++
.../src/main/python/core/models/schema/__init__.py | 2 +
.../python/core/models/schema/attribute_type.py | 2 +-
.../schema}/big_object_pointer.py | 2 +-
amber/src/main/python/core/models/tuple.py | 2 +-
.../python/core/storage/iceberg/iceberg_utils.py | 2 +-
amber/src/main/python/pytexera/__init__.py | 4 +-
.../pytexera/{ => storage}/big_object_manager.py | 19 ++--
bin/deployment/computing-unit-master.dockerfile | 4 +-
14 files changed, 319 insertions(+), 26 deletions(-)
diff --git a/amber/src/main/python/core/architecture/packaging/output_manager.py b/amber/src/main/python/core/architecture/packaging/output_manager.py
index eb027920ee..b2a6392dfe 100644
--- a/amber/src/main/python/core/architecture/packaging/output_manager.py
+++ b/amber/src/main/python/core/architecture/packaging/output_manager.py
@@ -259,7 +259,7 @@ class OutputManager:
)
def tuple_to_frame(self, tuples: typing.List[Tuple]) -> DataFrame:
- from core.storage.big_object_pointer import BigObjectPointer
+ from core.models.schema.big_object_pointer import BigObjectPointer
return DataFrame(
frame=Table.from_pydict(
diff --git a/amber/src/main/python/core/models/RTableExecutor.py b/amber/src/main/python/core/models/RTableExecutor.py
index db8610e6f2..a72451012b 100644
--- a/amber/src/main/python/core/models/RTableExecutor.py
+++ b/amber/src/main/python/core/models/RTableExecutor.py
@@ -23,7 +23,14 @@ from rpy2.robjects.conversion import localconverter as local_converter
from rpy2_arrow.arrow import rarrow_to_py_table, converter as arrow_converter
from typing import Iterator, Optional, Union
-from core.models import ArrowTableTupleProvider, Tuple, TupleLike, Table, TableLike
+from core.models import (
+ ArrowTableTupleProvider,
+ Tuple,
+ TupleLike,
+ Table,
+ TableLike,
+ r_utils,
+)
from core.models.operator import SourceOperator, TableOperator
@@ -45,6 +52,36 @@ class RTableExecutor(TableOperator):
"""
)
+ # BigObject column conversion helpers
+ _wrap_big_object_cols = robjects.r(
+ """
+ function(df, cols) {
+ for (col in cols) {
+ df[[col]] <- new_BigObjectPointerColumn(lapply(df[[col]], function(uri) {
+ if (is.na(uri) || is.null(uri) || nchar(uri) == 0) NA else BigObjectPointer$new(uri)
+ }))
+ }
+ df
+ }
+ """
+ )
+
+ _unwrap_big_object_cols = robjects.r(
+ """
+ function(df, cols) {
+ for (col in cols) {
+ vals <- if (inherits(df[[col]], "BigObjectPointerColumn")) unclass(df[[col]]) else df[[col]]
+ df[[col]] <- sapply(vals, function(obj) {
+ if (is.na(obj) || is.null(obj)) NA_character_
+ else if (inherits(obj, "BigObjectPointer")) obj$uri
+ else as.character(obj)
+ })
+ }
+ df
+ }
+ """
+ )
+
def __init__(self, r_code: str):
"""
Initialize the RTableExecutor with R code.
@@ -67,20 +104,71 @@ class RTableExecutor(TableOperator):
:return: Iterator[Optional[TableLike]], producing one TableLike object at a
time, or None.
"""
- input_pyarrow_table = pa.Table.from_pandas(table)
+ from core.models.schema.big_object_pointer import BigObjectPointer
+ from core.models.schema import Schema, AttributeType
+ from core.models.schema.attribute_type import FROM_ARROW_MAPPING
+
+ # Step 1: Identify and serialize BigObjectPointer → URI strings for input
+ input_big_object_cols = [
+ col
+ for col in table.columns
+ if len(table[col]) > 0 and isinstance(table[col].iloc[0], BigObjectPointer)
+ ]
+
+ if input_big_object_cols:
+ table = table.copy()
+ for col in input_big_object_cols:
+ table[col] = table[col].apply(
+ lambda x: x.uri if isinstance(x, BigObjectPointer) else x
+ )
+
+ # Step 2: Python → R conversion and execution
with local_converter(arrow_converter):
- input_r_dataframe = RTableExecutor._arrow_to_r_dataframe(
- input_pyarrow_table
+ # Convert pandas → Arrow → R dataframe
+ arrow_table = pa.Table.from_pandas(table)
+ r_df = RTableExecutor._arrow_to_r_dataframe(arrow_table)
+
+ # Wrap input BIG_OBJECT columns as BigObjectPointer for R UDF
+ if input_big_object_cols:
+ r_df = RTableExecutor._wrap_big_object_cols(r_df, input_big_object_cols)
+
+ # Execute user's R function
+ r_df = self._func(r_df, port)
+
+ # Detect which OUTPUT columns are BIG_OBJECT
+ output_big_object_cols = list(
+ robjects.r(
+ 'function(df) names(df)[sapply(df, inherits, "BigObjectPointerColumn")]'
+ )(r_df)
)
- output_r_dataframe = self._func(input_r_dataframe, port)
- output_rarrow_table = RTableExecutor._r_dataframe_to_arrow(
- output_r_dataframe
+
+ # Unwrap output BIG_OBJECT columns to URI strings for Python
+ if output_big_object_cols:
+ r_df = RTableExecutor._unwrap_big_object_cols(
+ r_df, output_big_object_cols
+ )
+
+ # Convert R dataframe → Arrow → pandas
+ arrow_table = rarrow_to_py_table(RTableExecutor._r_dataframe_to_arrow(r_df))
+
+ # Step 3: Build output schema with correct BIG_OBJECT type information
+ output_schema = Schema()
+ for col in arrow_table.column_names:
+ attr_type = (
+ AttributeType.BIG_OBJECT
+ if col in output_big_object_cols
+ else FROM_ARROW_MAPPING[arrow_table.schema.field(col).type.id]
)
- output_pyarrow_table = rarrow_to_py_table(output_rarrow_table)
+ output_schema.add(col, attr_type)
- for field_accessor in ArrowTableTupleProvider(output_pyarrow_table):
+ # Step 4: Create Arrow table with metadata and yield Tuples
+ arrow_with_metadata = pa.Table.from_pandas(
+ arrow_table.to_pandas(), schema=output_schema.as_arrow_schema()
+ )
+
+ for field_accessor in ArrowTableTupleProvider(arrow_with_metadata):
yield Tuple(
- {name: field_accessor for name in output_pyarrow_table.column_names}
+ {name: field_accessor for name in arrow_with_metadata.column_names}
)
diff --git a/amber/src/main/python/core/models/RTupleExecutor.py b/amber/src/main/python/core/models/RTupleExecutor.py
index 67f05fb2d3..3b0a966597 100644
--- a/amber/src/main/python/core/models/RTupleExecutor.py
+++ b/amber/src/main/python/core/models/RTupleExecutor.py
@@ -85,8 +85,17 @@ class RTupleExecutor(TupleOperatorV2):
non_binary_tuple_schema: pa.Schema = (
non_binary_tuple._schema.as_arrow_schema()
)
+
+ # Serialize BigObjectPointer to URI string for Arrow
+ from core.models.schema.big_object_pointer import BigObjectPointer
+
+ serialized_dict = {
+ k: (v.uri if isinstance(v, BigObjectPointer) else v)
+ for k, v in non_binary_tuple.as_dict().items()
+ }
+
non_binary_pyarrow_array: pa.StructArray = pa.array(
- [non_binary_tuple.as_dict()],
+ [serialized_dict],
type=pa.struct(non_binary_tuple_schema),
)
@@ -113,6 +122,25 @@ class RTupleExecutor(TupleOperatorV2):
)
)
+ # Wrap URI strings → R BigObjectPointer objects for BIG_OBJECT fields
+ from core.models.schema.attribute_type import AttributeType
+
+ pointer_class = robjects.r("BigObjectPointer")
+ wrapped = {
+ name: (
+ pointer_class(input_r_list.rx2(name)[0])
+ if (
+ input_r_list.rx2(name)
+ and isinstance(input_r_list.rx2(name)[0], str)
+ and tuple_._schema.get_attr_type(name)
+ == AttributeType.BIG_OBJECT
+ )
+ else input_r_list.rx2(name)
+ )
+ for name in input_r_list.names
+ }
+ input_r_list = robjects.vectors.ListVector(wrapped)
+
output_r_generator: rpy2.robjects.SignatureTranslatedFunction = self._func(
input_r_list, port
)
diff --git a/amber/src/main/python/core/models/big_object_manager.R b/amber/src/main/python/core/models/big_object_manager.R
new file mode 100644
index 0000000000..c365c11141
--- /dev/null
+++ b/amber/src/main/python/core/models/big_object_manager.R
@@ -0,0 +1,99 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# BigObjectManager - Manager for reading big objects from S3
+#
+# This file defines the BigObjectManager API and BigObjectStream class.
+# Users call BigObjectManager$open() in their R UDF code to read big objects.
+
+# Load required packages
+if (!require("aws.s3", quietly = TRUE)) {
+ warning("Package 'aws.s3' not installed. Install with: install.packages('aws.s3')")
+}
+
+# BigObjectStream Reference Class
+# Provides stream-like access to big object content
+BigObjectStream <- setRefClass("BigObjectStream",
+ fields = list(conn = "ANY", uri = "character", is_closed = "logical"),
+ methods = list(
+ initialize = function(raw_bytes, uri_val) {
+ conn <<- rawConnection(raw_bytes, open = "rb")
+ uri <<- uri_val
+ is_closed <<- FALSE
+ },
+ read = function(n = -1L) {
+ if (is_closed) stop("Stream is closed")
+ readBin(conn, "raw", if (n == -1L) 1e9 else n)
+ },
+ close = function() {
+ if (!is_closed) {
+ base::close(conn)
+ is_closed <<- TRUE
+ }
+ },
+ finalize = function() close()
+ )
+)
+
+# BigObjectManager API
+# Main interface for accessing big objects from S3
+BigObjectManager <- list(
+ open = function(pointer_or_uri) {
+ # Extract from list if needed (for backward compatibility)
+ if (is.list(pointer_or_uri) && length(pointer_or_uri) == 1) {
+ pointer_or_uri <- pointer_or_uri[[1]]
+ }
+
+ # Get URI string
+ uri <- if (inherits(pointer_or_uri, "BigObjectPointer")) {
+ pointer_or_uri$uri
+ } else if (is.character(pointer_or_uri)) {
+ pointer_or_uri
+ } else {
+ stop("Expected BigObjectPointer or character URI")
+ }
+
+ if (!grepl("^s3://", uri)) stop(paste("Invalid S3 URI:", uri))
+
+ # Parse s3://bucket/key
+ parts <- strsplit(sub("^s3://", "", uri), "/", fixed = TRUE)[[1]]
+ if (length(parts) < 2) stop(paste("Invalid S3 URI format:", uri))
+
+ # Configure S3 credentials from environment variables
+ Sys.setenv(
+ AWS_ACCESS_KEY_ID = Sys.getenv("STORAGE_S3_AUTH_USERNAME", "texera_minio"),
+ AWS_SECRET_ACCESS_KEY = Sys.getenv("STORAGE_S3_AUTH_PASSWORD", "password"),
+ AWS_S3_ENDPOINT = Sys.getenv("STORAGE_S3_ENDPOINT", "localhost:9000"),
+ AWS_DEFAULT_REGION = Sys.getenv("STORAGE_S3_REGION", "us-west-2")
+ )
+
+ # Fetch object from S3
+ raw_bytes <- tryCatch(
+ aws.s3::get_object(
+ object = paste(parts[-1], collapse = "/"),
+ bucket = parts[1],
+ region = Sys.getenv("AWS_DEFAULT_REGION"),
+ base_url = Sys.getenv("AWS_S3_ENDPOINT"),
+ use_https = grepl("^https://", Sys.getenv("AWS_S3_ENDPOINT"))
+ ),
+ error = function(e) stop(paste("Failed to open", uri, ":", conditionMessage(e)))
+ )
+
+ BigObjectStream$new(raw_bytes, uri)
+ }
+)
+
diff --git a/amber/src/main/python/core/models/big_object_pointer.R b/amber/src/main/python/core/models/big_object_pointer.R
new file mode 100644
index 0000000000..d75284a3af
--- /dev/null
+++ b/amber/src/main/python/core/models/big_object_pointer.R
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# BigObjectPointer - Reference to a large object stored in S3
+#
+# This file defines the BigObjectPointer Reference Class, which represents
+# a pointer to a big object. Users receive BigObjectPointer objects in their
+# R UDF code when working with BIG_OBJECT type fields.
+
+BigObjectPointer <- setRefClass("BigObjectPointer",
+ fields = list(uri = "character"),
+ methods = list(
+ initialize = function(uri_val) {
+ if (!grepl("^s3://", uri_val)) stop(paste("Invalid S3 URI:", uri_val))
+ uri <<- uri_val
+ }
+ )
+)
+
+# BigObjectPointerColumn - S3 class for natural table$col[i] syntax
+#
+# R stores object columns as list-columns (require [[i]]), but we want [i] to work
+# like other columns. This S3 class overrides [ to return elements directly.
+
+new_BigObjectPointerColumn <- function(x) {
+ structure(x, class = c("BigObjectPointerColumn", "list"))
+}
+
+`[.BigObjectPointerColumn` <- function(x, i) {
+ result <- unclass(x)[i]
+ if (length(i) == 1) result[[1]] else new_BigObjectPointerColumn(result)
+}
+
+`[[.BigObjectPointerColumn` <- function(x, i) unclass(x)[[i]]
+
+length.BigObjectPointerColumn <- function(x) length(unclass(x))
+
diff --git a/amber/src/main/python/core/models/r_utils.py b/amber/src/main/python/core/models/r_utils.py
index d1f6e512bd..f0516cab99 100644
--- a/amber/src/main/python/core/models/r_utils.py
+++ b/amber/src/main/python/core/models/r_utils.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+import os
import rpy2
import rpy2.rinterface as rinterface
import rpy2.robjects as robjects
@@ -24,6 +25,15 @@ from core.models import Tuple
warnings.filterwarnings(action="ignore", category=UserWarning, module=r"rpy2*")
+# Auto-load R BigObject user-facing API when this module is imported
+# Load order: pointer → manager (dependency order)
+_r_files = [
+ os.path.join(os.path.dirname(__file__), "big_object_pointer.R"),
+ os.path.join(os.path.dirname(__file__), "big_object_manager.R"),
+]
+for r_file in _r_files:
+ robjects.r(f'source("{r_file}")')
+
def convert_r_to_py(value: rpy2.robjects):
"""
@@ -31,6 +41,14 @@ def convert_r_to_py(value: rpy2.robjects):
:return: A Python representation of the value, if convertable.
If not, it returns the value itself
"""
+ # Convert R BigObjectPointer to Python BigObjectPointer
+ if hasattr(value, "rclass") and value.rclass and "BigObjectPointer" in value.rclass:
+ from core.models.schema.big_object_pointer import BigObjectPointer
+
+ uri_field = robjects.r("function(obj) obj$uri")(value)
+ uri = str(uri_field[0]) if len(uri_field) > 0 else str(uri_field)
+ return BigObjectPointer(uri)
+
if isinstance(value, robjects.vectors.BoolVector):
return bool(value[0])
if isinstance(value, robjects.vectors.IntVector):
diff --git a/amber/src/main/python/core/models/schema/__init__.py b/amber/src/main/python/core/models/schema/__init__.py
index 7c8b57bb31..a21430a3d2 100644
--- a/amber/src/main/python/core/models/schema/__init__.py
+++ b/amber/src/main/python/core/models/schema/__init__.py
@@ -16,12 +16,14 @@
# under the License.
from .attribute_type import AttributeType
+from .big_object_pointer import BigObjectPointer
from .field import Field
from .schema import Schema
__all__ = [
"AttributeType",
+ "BigObjectPointer",
"Field",
"Schema",
]
diff --git a/amber/src/main/python/core/models/schema/attribute_type.py b/amber/src/main/python/core/models/schema/attribute_type.py
index 63c952d1f8..f3f6e50f38 100644
--- a/amber/src/main/python/core/models/schema/attribute_type.py
+++ b/amber/src/main/python/core/models/schema/attribute_type.py
@@ -20,7 +20,7 @@ import pyarrow as pa
from bidict import bidict
from enum import Enum
from pyarrow import lib
-from core.storage.big_object_pointer import BigObjectPointer
+from core.models.schema.big_object_pointer import BigObjectPointer
class AttributeType(Enum):
diff --git a/amber/src/main/python/core/storage/big_object_pointer.py b/amber/src/main/python/core/models/schema/big_object_pointer.py
similarity index 95%
rename from amber/src/main/python/core/storage/big_object_pointer.py
rename to amber/src/main/python/core/models/schema/big_object_pointer.py
index 4aac9d3a63..e846f8411d 100644
--- a/amber/src/main/python/core/storage/big_object_pointer.py
+++ b/amber/src/main/python/core/models/schema/big_object_pointer.py
@@ -17,7 +17,7 @@
"""
BigObjectPointer represents a reference to a large object stored externally (e.g., S3).
-This is a storage reference class used throughout the system for handling big objects.
+This is a schema type class used throughout the system for handling BIG_OBJECT attribute types.
"""
diff --git a/amber/src/main/python/core/models/tuple.py b/amber/src/main/python/core/models/tuple.py
index f88d75b4fe..6369b6dba2 100644
--- a/amber/src/main/python/core/models/tuple.py
+++ b/amber/src/main/python/core/models/tuple.py
@@ -29,7 +29,7 @@ from pympler import asizeof
from typing import Any, List, Iterator, Callable
from typing_extensions import Protocol, runtime_checkable
-from core.storage.big_object_pointer import BigObjectPointer
+from core.models.schema.big_object_pointer import BigObjectPointer
from .schema.attribute_type import TO_PYOBJECT_MAPPING, AttributeType
from .schema.field import Field
from .schema.schema import Schema
diff --git a/amber/src/main/python/core/storage/iceberg/iceberg_utils.py b/amber/src/main/python/core/storage/iceberg/iceberg_utils.py
index 3c23df73b0..c8e0db9e93 100644
--- a/amber/src/main/python/core/storage/iceberg/iceberg_utils.py
+++ b/amber/src/main/python/core/storage/iceberg/iceberg_utils.py
@@ -243,7 +243,7 @@ def amber_tuples_to_arrow_table(
Converts a list of amber tuples to a pyarrow table for serialization.
Handles BIG_OBJECT field name encoding and serialization.
"""
- from core.storage.big_object_pointer import BigObjectPointer
+ from core.models.schema.big_object_pointer import BigObjectPointer
# Build data dict using Iceberg schema field names (encoded)
data_dict = {}
diff --git a/amber/src/main/python/pytexera/__init__.py b/amber/src/main/python/pytexera/__init__.py
index 2b3d59be10..f65eefac9d 100644
--- a/amber/src/main/python/pytexera/__init__.py
+++ b/amber/src/main/python/pytexera/__init__.py
@@ -21,13 +21,14 @@ from typing import Iterator, Optional, Union
from pyamber import *
from .storage.dataset_file_document import DatasetFileDocument
+from .storage.big_object_manager import BigObjectManager
from .udf.udf_operator import (
UDFOperatorV2,
UDFTableOperator,
UDFBatchOperator,
UDFSourceOperator,
)
-from .big_object_manager import BigObjectManager
+from core.models.schema.big_object_pointer import BigObjectPointer
__all__ = [
"State",
@@ -43,6 +44,7 @@ __all__ = [
"UDFSourceOperator",
"DatasetFileDocument",
"BigObjectManager",
+ "BigObjectPointer",
# export external tools to be used
"overrides",
"logger",
diff --git a/amber/src/main/python/pytexera/big_object_manager.py b/amber/src/main/python/pytexera/storage/big_object_manager.py
similarity index 88%
rename from amber/src/main/python/pytexera/big_object_manager.py
rename to amber/src/main/python/pytexera/storage/big_object_manager.py
index f61549fb5f..a59129c2c9 100644
--- a/amber/src/main/python/pytexera/big_object_manager.py
+++ b/amber/src/main/python/pytexera/storage/big_object_manager.py
@@ -19,36 +19,39 @@
import os
from typing import BinaryIO
-from core.storage.big_object_pointer import BigObjectPointer
+from core.models.schema.big_object_pointer import BigObjectPointer
class BigObjectStream:
- """Stream for reading big objects (matches Scala BigObjectStream)."""
+ """Stream for reading big objects (matches Scala/R BigObjectStream)."""
def __init__(self, body: BinaryIO, pointer: BigObjectPointer):
self._body = body
self._pointer = pointer
self._closed = False
- def read(self, amt=None):
+ def read(self, n: int = -1) -> bytes:
+ """Read n bytes from stream (-1 = read all)."""
if self._closed:
raise ValueError("I/O operation on closed stream")
- return self._body.read(amt)
+ if n == -1:
+ return self._body.read()
+ return self._body.read(n)
def close(self):
+ """Close the stream."""
if not self._closed:
self._closed = True
self._body.close()
def __enter__(self):
+ """Context manager entry."""
return self
def __exit__(self, exc_type, exc_val, exc_tb):
+ """Context manager exit - automatically cleanup."""
self.close()
-
- @property
- def closed(self):
- return self._closed
+ return False
class BigObjectManager:
diff --git a/bin/deployment/computing-unit-master.dockerfile b/bin/deployment/computing-unit-master.dockerfile
index d44fe9703d..74e0d52e71 100644
--- a/bin/deployment/computing-unit-master.dockerfile
+++ b/bin/deployment/computing-unit-master.dockerfile
@@ -93,7 +93,9 @@ RUN Rscript -e "options(repos = c(CRAN = 'https://cran.r-project.org')); \
install.packages('remotes'); \
remotes::install_version('arrow', version='14.0.2.1', \
repos='https://cran.r-project.org', upgrade='never'); \
- cat('R arrow version: ', as.character(packageVersion('arrow')), '\n')"
+ cat('R arrow version: ', as.character(packageVersion('arrow')), '\n')" && \
+ Rscript -e "options(repos = c(CRAN = 'https://cran.r-project.org')); \
+ install.packages('aws.s3')";
ENV LD_LIBRARY_PATH=/usr/local/lib/R/lib:$LD_LIBRARY_PATH
# Copy the built texera binary from the build phase
(texera) 07/07: create() API for python + cleanup the parameters
Posted by ku...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kunwp1 pushed a commit to branch chris-big-object-version-0
in repository https://gitbox.apache.org/repos/asf/texera.git
commit cea73acbb607f0d6fc6712372b009111e2f24d0f
Author: Kunwoo Park <ku...@uci.edu>
AuthorDate: Mon Oct 27 13:08:37 2025 -0700
create() API for python + cleanup the parameters
---
.../control/initialize_executor_handler.py | 5 ++
.../python/core/architecture/managers/context.py | 13 ++++
.../architecture/managers/execution_context.py | 62 +++++++++++++++
.../main/python/core/runnables/data_processor.py | 5 ++
.../amber/engine/architecture/rpc/__init__.py | 1 +
.../python/pytexera/storage/big_object_manager.py | 91 +++++++++++-----------
.../texera/service/util/BigObjectManager.scala | 25 +++---
.../source/scan/FileScanSourceOpExec.scala | 13 +---
8 files changed, 148 insertions(+), 67 deletions(-)
diff --git a/amber/src/main/python/core/architecture/handlers/control/initialize_executor_handler.py b/amber/src/main/python/core/architecture/handlers/control/initialize_executor_handler.py
index 0d86dbc212..45e992358b 100644
--- a/amber/src/main/python/core/architecture/handlers/control/initialize_executor_handler.py
+++ b/amber/src/main/python/core/architecture/handlers/control/initialize_executor_handler.py
@@ -28,6 +28,11 @@ class InitializeExecutorHandler(ControlHandler):
async def initialize_executor(self, req: InitializeExecutorRequest) -> EmptyReturn:
op_exec_with_code: OpExecWithCode = get_one_of(req.op_exec_init_info)
+
+ # Store execution_id in context (from InitializeExecutorRequest)
+ # This is the real execution ID, not the workflow ID
+ self.context.execution_id = req.execution_id
+
self.context.executor_manager.initialize_executor(
op_exec_with_code.code, req.is_source, op_exec_with_code.language
)
diff --git a/amber/src/main/python/core/architecture/managers/context.py b/amber/src/main/python/core/architecture/managers/context.py
index 826006ccb6..fa92a6db26 100644
--- a/amber/src/main/python/core/architecture/managers/context.py
+++ b/amber/src/main/python/core/architecture/managers/context.py
@@ -16,6 +16,7 @@
# under the License.
from typing import Optional
+import re
from proto.org.apache.amber.core import ActorVirtualIdentity, ChannelIdentity
from proto.org.apache.amber.engine.architecture.worker import WorkerState
@@ -23,6 +24,7 @@ from .console_message_manager import ConsoleMessageManager
from .debug_manager import DebugManager
from .embedded_control_message_manager import EmbeddedControlMessageManager
from .exception_manager import ExceptionManager
+from .execution_context import ExecutionContext
from .executor_manager import ExecutorManager
from .pause_manager import PauseManager
from .state_manager import StateManager
@@ -46,6 +48,17 @@ class Context:
def __init__(self, worker_id, input_queue):
self.worker_id = worker_id
self.input_queue: InternalQueue = input_queue
+
+ # execution_id: set by InitializeExecutorHandler from InitializeExecutorRequest
+ # operator_id: parsed from worker_id (format: Worker:WF{wid}-{opid}-{layer}-{idx})
+ self.execution_id: Optional[int] = None
+ self.operator_id: Optional[str] = None
+
+ # Extract operator_id from worker_id (e.g., "Worker:WF78-op-scan-main-0" → "op-scan")
+ match = re.match(r"Worker:WF\d+-(.+)-\w+-\d+", worker_id)
+ if match:
+ self.operator_id = match.group(1)
+
self.executor_manager = ExecutorManager()
self.current_input_channel_id: Optional[ChannelIdentity] = None
self.tuple_processing_manager = TupleProcessingManager()
diff --git a/amber/src/main/python/core/architecture/managers/execution_context.py b/amber/src/main/python/core/architecture/managers/execution_context.py
new file mode 100644
index 0000000000..31e02436a6
--- /dev/null
+++ b/amber/src/main/python/core/architecture/managers/execution_context.py
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+ExecutionContext provides thread-local storage for execution-related information.
+This allows BigObjectManager and other utilities to automatically access
+execution_id and operator_id without requiring explicit parameters.
+"""
+
+import threading
+from typing import Optional
+
+
+class ExecutionContext:
+ """
+ Thread-local storage for execution context information.
+ Similar to Scala's ExecutionContext.
+ """
+
+ _thread_local = threading.local()
+
+ @classmethod
+ def set_execution_id(cls, execution_id: int) -> None:
+ """Set the execution ID for the current thread."""
+ cls._thread_local.execution_id = execution_id
+
+ @classmethod
+ def get_execution_id(cls) -> Optional[int]:
+ """Get the execution ID for the current thread, or None if not set."""
+ return getattr(cls._thread_local, "execution_id", None)
+
+ @classmethod
+ def set_operator_id(cls, operator_id: str) -> None:
+ """Set the operator ID for the current thread."""
+ cls._thread_local.operator_id = operator_id
+
+ @classmethod
+ def get_operator_id(cls) -> Optional[str]:
+ """Get the operator ID for the current thread, or None if not set."""
+ return getattr(cls._thread_local, "operator_id", None)
+
+ @classmethod
+ def clear(cls) -> None:
+ """Clear all execution context information for the current thread."""
+ if hasattr(cls._thread_local, "execution_id"):
+ delattr(cls._thread_local, "execution_id")
+ if hasattr(cls._thread_local, "operator_id"):
+ delattr(cls._thread_local, "operator_id")
diff --git a/amber/src/main/python/core/runnables/data_processor.py b/amber/src/main/python/core/runnables/data_processor.py
index ce8f937142..0c9747a406 100644
--- a/amber/src/main/python/core/runnables/data_processor.py
+++ b/amber/src/main/python/core/runnables/data_processor.py
@@ -23,6 +23,7 @@ from threading import Event
from typing import Iterator, Optional
from core.architecture.managers import Context
+from core.architecture.managers.execution_context import ExecutionContext
from core.models import ExceptionInfo, State, TupleLike, InternalMarker
from core.models.internal_marker import StartChannel, EndChannel
from core.models.table import all_output_to_tuple
@@ -51,6 +52,10 @@ class DataProcessor(Runnable, Stoppable):
self._running.set()
self._switch_context()
while self._running.is_set():
+ # Sync ExecutionContext from context (for thread-local storage)
+ ExecutionContext.set_execution_id(self._context.execution_id)
+ ExecutionContext.set_operator_id(self._context.operator_id)
+
marker = self._context.tuple_processing_manager.get_internal_marker()
state = self._context.state_processing_manager.get_input_state()
tuple_ = self._context.tuple_processing_manager.current_input_tuple
diff --git a/amber/src/main/python/proto/org/apache/amber/engine/architecture/rpc/__init__.py b/amber/src/main/python/proto/org/apache/amber/engine/architecture/rpc/__init__.py
index 0a913e8b88..258fc928f7 100644
--- a/amber/src/main/python/proto/org/apache/amber/engine/architecture/rpc/__init__.py
+++ b/amber/src/main/python/proto/org/apache/amber/engine/architecture/rpc/__init__.py
@@ -378,6 +378,7 @@ class InitializeExecutorRequest(betterproto.Message):
total_worker_count: int = betterproto.int32_field(1)
op_exec_init_info: "___core__.OpExecInitInfo" = betterproto.message_field(2)
is_source: bool = betterproto.bool_field(3)
+ execution_id: int = betterproto.int64_field(4)
@dataclass(eq=False, repr=False)
diff --git a/amber/src/main/python/pytexera/storage/big_object_manager.py b/amber/src/main/python/pytexera/storage/big_object_manager.py
index c116c1e392..2fa2c7398c 100644
--- a/amber/src/main/python/pytexera/storage/big_object_manager.py
+++ b/amber/src/main/python/pytexera/storage/big_object_manager.py
@@ -19,10 +19,11 @@
import time
import uuid
-from typing import BinaryIO, Union
+from typing import BinaryIO, Union, Optional
from io import BytesIO
from core.models.schema.big_object_pointer import BigObjectPointer
from core.storage.storage_config import StorageConfig
+from core.architecture.managers.execution_context import ExecutionContext
class BigObjectStream:
@@ -95,18 +96,31 @@ class BigObjectManager:
if cls._db_connection is None:
try:
import psycopg2
- # Parse JDBC URL to extract host, port, database
- # Format: jdbc:postgresql://host:port/database?params
+
+ # Parse JDBC URL to extract host, port, database, and schema
+ # Format: jdbc:postgresql://host:port/database?currentSchema=texera_db,public
jdbc_url = StorageConfig.JDBC_URL
if jdbc_url.startswith("jdbc:postgresql://"):
url_part = jdbc_url.replace("jdbc:postgresql://", "")
# Split by '?' to separate params
- url_without_params = url_part.split("?")[0]
+ url_parts = url_part.split("?")
+ url_without_params = url_parts[0]
+
+ # Parse schema from params (e.g., currentSchema=texera_db,public)
+ schema = "texera_db" # default
+ if len(url_parts) > 1:
+ params = url_parts[1]
+ for param in params.split("&"):
+ if param.startswith("currentSchema="):
+ schema_value = param.split("=")[1]
+ # Take first schema from comma-separated list
+ schema = schema_value.split(",")[0]
+
# Split by '/' to get host:port and database
parts = url_without_params.split("/")
host_port = parts[0]
database = parts[1] if len(parts) > 1 else "texera_db"
-
+
# Split host and port
if ":" in host_port:
host, port = host_port.split(":")
@@ -114,18 +128,22 @@ class BigObjectManager:
else:
host = host_port
port = 5432
-
+
+ # Connect and set search_path
cls._db_connection = psycopg2.connect(
host=host,
port=port,
user=StorageConfig.JDBC_USERNAME,
password=StorageConfig.JDBC_PASSWORD,
- database=database
+ database=database,
+ options=f"-c search_path={schema},public",
)
else:
raise ValueError(f"Invalid JDBC URL format: {jdbc_url}")
except ImportError:
- raise RuntimeError("psycopg2 required. Install with: pip install psycopg2-binary")
+ raise RuntimeError(
+ "psycopg2 required. Install with: pip install psycopg2-binary"
+ )
except Exception as e:
raise RuntimeError(f"Failed to connect to database: {e}")
return cls._db_connection
@@ -140,57 +158,43 @@ class BigObjectManager:
# Bucket doesn't exist, create it
try:
s3.create_bucket(Bucket=bucket)
- print(f"Created S3 bucket: {bucket}")
except Exception as e:
raise RuntimeError(f"Failed to create bucket {bucket}: {e}")
@classmethod
- def create(cls, execution_id: int, operator_id: str, data: Union[bytes, BinaryIO]) -> BigObjectPointer:
+ def create(cls, data: Union[bytes, BinaryIO]) -> BigObjectPointer:
"""
- Creates a big object from bytes or a stream and uploads to S3.
- Registers the big object in the database for cleanup tracking.
+ Creates a big object from bytes or stream, uploads to S3, and registers in database.
+ Automatically retrieves execution_id and operator_id from ExecutionContext.
Args:
- execution_id: The execution ID this big object belongs to.
- operator_id: The operator ID that created this big object.
- data: Either bytes or a file-like object (stream) containing the data.
+ data: Bytes or file-like object containing the data.
Returns:
- A BigObjectPointer that can be used in tuples.
-
- Usage:
- # From bytes
- pointer = BigObjectManager.create(execution_id, operator_id, b"large data...")
+ BigObjectPointer for use in tuples.
+ """
+ # Get IDs from ExecutionContext
+ execution_id = ExecutionContext.get_execution_id()
+ operator_id = ExecutionContext.get_operator_id()
- # From file
- with open("large_file.bin", "rb") as f:
- pointer = BigObjectManager.create(execution_id, operator_id, f)
+ if execution_id is None or operator_id is None:
+ raise RuntimeError(
+ "ExecutionContext not initialized. This should not happen in normal workflow execution."
+ )
- # From BytesIO
- buffer = BytesIO(b"data...")
- pointer = BigObjectManager.create(execution_id, operator_id, buffer)
- """
cls._create_bucket_if_not_exist(cls.DEFAULT_BUCKET)
- # Generate unique object key
- timestamp = int(time.time() * 1000) # milliseconds
- unique_id = str(uuid.uuid4())
- object_key = f"{timestamp}/{unique_id}"
+ # Generate unique S3 key and URI
+ object_key = f"{int(time.time() * 1000)}/{uuid.uuid4()}"
uri = f"s3://{cls.DEFAULT_BUCKET}/{object_key}"
-
s3 = cls._get_s3_client()
+ # Upload to S3
try:
- # Upload to S3
if isinstance(data, bytes):
- # Upload bytes directly
s3.put_object(Bucket=cls.DEFAULT_BUCKET, Key=object_key, Body=data)
else:
- # Upload from stream (file-like object)
s3.upload_fileobj(data, cls.DEFAULT_BUCKET, object_key)
-
- print(f"Uploaded big object to S3: {uri}")
-
except Exception as e:
raise RuntimeError(f"Failed to upload big object to S3: {e}")
@@ -200,19 +204,16 @@ class BigObjectManager:
cursor = conn.cursor()
cursor.execute(
"INSERT INTO big_object (execution_id, operator_id, uri) VALUES (%s, %s, %s)",
- (execution_id, operator_id, uri)
+ (execution_id, operator_id, uri),
)
conn.commit()
cursor.close()
- print(f"Registered big object in database: eid={execution_id}, opid={operator_id}, uri={uri}")
-
except Exception as e:
- # Database registration failed - clean up S3 object
- print(f"Failed to register big object in database, cleaning up: {uri}")
+ # Clean up S3 object if database registration fails
try:
s3.delete_object(Bucket=cls.DEFAULT_BUCKET, Key=object_key)
- except Exception as cleanup_error:
- print(f"Failed to cleanup orphaned S3 object {uri}: {cleanup_error}")
+ except:
+ pass
raise RuntimeError(f"Failed to create big object: {e}")
return BigObjectPointer(uri)
diff --git a/common/workflow-core/src/main/scala/org/apache/texera/service/util/BigObjectManager.scala b/common/workflow-core/src/main/scala/org/apache/texera/service/util/BigObjectManager.scala
index 6fcdb7b0e5..a7a3d1b2b3 100644
--- a/common/workflow-core/src/main/scala/org/apache/texera/service/util/BigObjectManager.scala
+++ b/common/workflow-core/src/main/scala/org/apache/texera/service/util/BigObjectManager.scala
@@ -20,6 +20,7 @@
package org.apache.texera.service.util
import com.typesafe.scalalogging.LazyLogging
+import org.apache.amber.core.executor.ExecutionContext
import org.apache.amber.core.tuple.BigObjectPointer
import org.apache.texera.dao.SqlServer
import org.apache.texera.dao.jooq.generated.Tables.BIG_OBJECT
@@ -74,16 +75,20 @@ object BigObjectManager extends LazyLogging {
private lazy val context = SqlServer.getInstance().createDSLContext()
/**
- * Creates a big object from an InputStream using multipart upload.
- * Handles streams of any size without loading into memory.
- * Registers the big object in the database for cleanup tracking.
+ * Creates a big object from InputStream, uploads to S3, and registers in database.
+ * Automatically retrieves execution ID and operator ID from ExecutionContext.
*
- * @param executionId The execution ID this big object belongs to.
- * @param operatorId The operator ID that created this big object.
- * @param stream The input stream containing the big object data.
- * @return A BigObjectPointer that can be used in tuples.
+ * @param stream Input stream containing the data.
+ * @return BigObjectPointer for use in tuples.
*/
- def create(executionId: Int, operatorId: String, stream: InputStream): BigObjectPointer = {
+ def create(stream: InputStream): BigObjectPointer = {
+ val executionId = ExecutionContext.getExecutionId.getOrElse(
+ throw new IllegalStateException("Execution ID not set in ExecutionContext")
+ )
+ val operatorId = ExecutionContext.getOperatorId.getOrElse(
+ throw new IllegalStateException("Operator ID not set in ExecutionContext")
+ )
+
S3StorageClient.createBucketIfNotExist(DEFAULT_BUCKET)
val objectKey = s"${System.currentTimeMillis()}/${UUID.randomUUID()}"
@@ -102,13 +107,11 @@ object BigObjectManager extends LazyLogging {
logger.debug(s"Registered big object: eid=$executionId, opid=$operatorId, uri=$uri")
} catch {
case e: Exception =>
- // Database failed - clean up S3 object
logger.error(s"Failed to register big object, cleaning up: $uri", e)
try {
S3StorageClient.deleteObject(DEFAULT_BUCKET, objectKey)
} catch {
- case cleanupError: Exception =>
- logger.error(s"Failed to cleanup orphaned S3 object: $uri", cleanupError)
+ case _: Exception => // Best effort cleanup
}
throw new RuntimeException(s"Failed to create big object: ${e.getMessage}", e)
}
diff --git a/common/workflow-operator/src/main/scala/org/apache/amber/operator/source/scan/FileScanSourceOpExec.scala b/common/workflow-operator/src/main/scala/org/apache/amber/operator/source/scan/FileScanSourceOpExec.scala
index 5013289c44..a9635ed668 100644
--- a/common/workflow-operator/src/main/scala/org/apache/amber/operator/source/scan/FileScanSourceOpExec.scala
+++ b/common/workflow-operator/src/main/scala/org/apache/amber/operator/source/scan/FileScanSourceOpExec.scala
@@ -19,7 +19,7 @@
package org.apache.amber.operator.source.scan
-import org.apache.amber.core.executor.{ExecutionContext, SourceOperatorExecutor}
+import org.apache.amber.core.executor.SourceOperatorExecutor
import org.apache.amber.core.storage.DocumentFactory
import org.apache.amber.core.tuple.AttributeTypeUtils.parseField
import org.apache.amber.core.tuple.TupleLike
@@ -87,16 +87,7 @@ class FileScanSourceOpExec private[scan] (
new String(toByteArray(entry), desc.fileEncoding.getCharset)
case FileAttributeType.BIG_OBJECT =>
// For big objects, create a big object pointer from the input stream
- // Get execution ID and operator ID from thread-local context
- val executionId = ExecutionContext.getExecutionId
- .getOrElse(
- throw new IllegalStateException("Execution ID not set in ExecutionContext")
- )
- val operatorId = ExecutionContext.getOperatorId
- .getOrElse(
- throw new IllegalStateException("Operator ID not set in ExecutionContext")
- )
- BigObjectManager.create(executionId, operatorId, entry)
+ BigObjectManager.create(entry)
case _ => parseField(toByteArray(entry), desc.attributeType.getType)
})
TupleLike(fields.toSeq: _*)