You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@graphar.apache.org by xi...@apache.org on 2025/10/29 06:58:20 UTC
(incubator-graphar) branch kuzu-extension updated: feat(c++): kuzu extension to import/export graphar data (#776)
This is an automated email from the ASF dual-hosted git repository.
xiaokang pushed a commit to branch kuzu-extension
in repository https://gitbox.apache.org/repos/asf/incubator-graphar.git
The following commit(s) were added to refs/heads/kuzu-extension by this push:
new 736795e9 feat(c++): kuzu extension to import/export graphar data (#776)
736795e9 is described below
commit 736795e9489052c507cc02d7fb9e797fb49a663b
Author: Gary <ga...@foxmail.com>
AuthorDate: Wed Oct 29 14:58:16 2025 +0800
feat(c++): kuzu extension to import/export graphar data (#776)
* feat: kuzu extension to import/export graphar data
* feat(c++): add test demo for graphar extension
* feat: add export to graphar and show graphar metadata
* fix: metadata code
* fix comments
---
cpp/extensions/kuzu-extension/CMakeLists.txt | 36 ++
cpp/extensions/kuzu-extension/README.md | 118 +++++
.../kuzu-extension/src/function/CMakeLists.txt | 11 +
.../kuzu-extension/src/function/graphar_export.cpp | 404 ++++++++++++++++
.../src/function/graphar_export_bindfunc.cpp | 503 ++++++++++++++++++++
.../src/function/graphar_metadata.cpp | 417 +++++++++++++++++
.../kuzu-extension/src/function/graphar_scan.cpp | 181 +++++++
.../src/function/graphar_scan_bindfunc.cpp | 518 +++++++++++++++++++++
.../src/include/function/graphar_export.h | 349 ++++++++++++++
.../src/include/function/graphar_metadata.h | 91 ++++
.../src/include/function/graphar_scan.h | 117 +++++
.../src/include/main/graphar_extension.h | 17 +
.../src/include/utils/graphar_utils.h | 86 ++++
.../kuzu-extension/src/main/CMakeLists.txt | 7 +
.../kuzu-extension/src/main/graphar_extension.cpp | 39 ++
.../kuzu-extension/src/utils/CMakeLists.txt | 7 +
.../kuzu-extension/src/utils/graphar_utils.cpp | 159 +++++++
cpp/extensions/kuzu-extension/test/README.md | 19 +
.../test/graphar_extension_copy_test.cpp | 121 +++++
.../test/graphar_extension_copy_to_test.cpp | 139 ++++++
.../test/graphar_extension_load_test.cpp | 66 +++
.../test/graphar_extension_metadata_test.cpp | 88 ++++
testing | 1 -
23 files changed, 3493 insertions(+), 1 deletion(-)
diff --git a/cpp/extensions/kuzu-extension/CMakeLists.txt b/cpp/extensions/kuzu-extension/CMakeLists.txt
new file mode 100644
index 00000000..5e7d1354
--- /dev/null
+++ b/cpp/extensions/kuzu-extension/CMakeLists.txt
@@ -0,0 +1,36 @@
+if (WIN32 OR BUILD_STATIC_EXTENSION)
+ set(GraphAr_USE_STATIC_LIBS ON)
+ set(Arrow_USE_STATIC_LIBS ON)
+else()
+ set(GraphAr_USE_STATIC_LIBS OFF)
+ set(Arrow_USE_STATIC_LIBS OFF)
+endif()
+
+find_package(graphar REQUIRED)
+find_package(Arrow REQUIRED)
+
+message(STATUS "Found GraphAr : ${graphar_VERSION}")
+message(STATUS "Found Arrow : ${Arrow_VERSION}")
+
+include_directories(
+ ${PROJECT_SOURCE_DIR}/src/include
+ ${CMAKE_BINARY_DIR}/src/include
+ ${PROJECT_SOURCE_DIR}/extension/graphar/src/include
+ src/include
+)
+
+add_subdirectory(src/function)
+add_subdirectory(src/main)
+
+build_extension_lib(${BUILD_STATIC_EXTENSION} "graphar")
+
+target_link_libraries(
+ kuzu_${EXTENSION_LIB_NAME}_extension
+ PUBLIC
+ graphar
+ # Arrow::arrow_shared
+)
+
+if (APPLE AND NOT BUILD_STATIC_EXTENSION)
+ set_apple_dynamic_lookup(kuzu_${EXTENSION_LIB_NAME}_extension)
+endif()
diff --git a/cpp/extensions/kuzu-extension/README.md b/cpp/extensions/kuzu-extension/README.md
new file mode 100644
index 00000000..0c2c2f09
--- /dev/null
+++ b/cpp/extensions/kuzu-extension/README.md
@@ -0,0 +1,118 @@
+# Kuzu — GraphAr Extension
+
+**API:** C++
+**Status:** first-cut, pragmatic implementation (ingest, export, metadata inspection)
+**Repository branch:** [https://github.com/apache/incubator-graphar/tree/kuzu-extension](https://github.com/apache/incubator-graphar/tree/kuzu-extension)
+
+---
+
+## TL;DR
+
+A pragmatic GraphAr extension for Kuzu implemented in C++ and built on GraphAr + Arrow C++ SDKs. The extension provides a translation layer that enables GraphAr datasets to be consumed and produced by Kuzu workflows. Supported operations include reading GraphAr datasets into Kuzu (`LOAD` / `COPY FROM`), inspecting GraphAr YAML metadata from within Kuzu, and exporting Kuzu query results into GraphAr-layout files (`COPY ... TO`). These capabilities are designed as pragmatic, cohesive featu [...]
+
+---
+
+## Context
+
+GraphAr is a property-graph interchange format that expresses dataset metadata in YAML and stores columnar payloads via Arrow/Parquet. This extension implements a translation-and-IO layer that parses GraphAr YAML, maps Arrow/GraphAr schema to Kuzu types, and performs parallel record-batch conversions so Kuzu can ingest and emit GraphAr-layout data without a separate conversion step.
+
+---
+
+## Implemented features
+
+The extension implements the following capabilities as a unified set:
+
+* GraphAr YAML schema parsing (node/edge tables, property columns, Arrow/Parquet descriptors).
+* Conservative mapping of GraphAr/Arrow column types to Kuzu primitive types.
+* Parallel RecordBatch readers that convert GraphAr data into Kuzu insert streams for `LOAD` / `COPY FROM` ingestion.
+* `COPY <Table> FROM "<graph.yml>" (file_format="graphar", table_name="<name>")` — import GraphAr data into an existing Kuzu table.
+* `COPY (<query>) TO "<out.graph.yml.graphar>" (table_name="<name>", target_dir="<dir>")` — export query results into GraphAr-layout files (YAML + Arrow/Parquet payloads).
+* `CALL GRAPHAR_METADATA('<path>', [vertex_type := '<v>'], [edge_type := '<e>'])` — read and return GraphAr YAML metadata entries from within Kuzu.
+* Non-invasive extension architecture: translation and IO logic implemented to minimize changes to Kuzu core.
+
+These capabilities are intended to be used together: inspect metadata, ingest GraphAr data, query and validate in Kuzu, and export GraphAr-layout outputs for downstream consumers.
+
+---
+
+## Usage examples
+
+```sql
+-- Inspect graph metadata
+CALL GRAPHAR_METADATA('/path/to/graph.graph.yml') RETURN *;
+
+-- Create Kuzu target table
+CREATE NODE TABLE Person(id INT64, firstName STRING, lastName STRING, gender STRING, PRIMARY KEY (id));
+
+-- Ingest GraphAr dataset into Kuzu table
+COPY Person FROM "/path/to/graph.graph.yml" (file_format="graphar", table_name="person");
+
+-- Export query results into GraphAr layout
+COPY (
+ MATCH (p:Person)
+ RETURN p.id AS id, p.firstName AS firstName, p.lastName AS lastName, p.gender AS gender
+ ORDER BY p.internal_id ASC
+) TO "/path/to/out.graph.yml.graphar" (table_name="person", target_dir="/some/output/dir/");
+```
+
+Notes on parameters:
+
+* `file_format="graphar"` selects this extension as the reader for `COPY FROM`.
+* `table_name` must match the GraphAr node/edge table name in the YAML.
+* `target_dir` (for `COPY TO`) is the directory where binary Arrow/Parquet payloads are written; the `TO` path serves as the logical YAML output name.
+* For edge export, queries typically return `from`/`to` (often using `internal_id`) to map edge endpoints.
+
+---
+
+## Design principles
+
+* **Translation layer (non-invasive):** keep modifications to Kuzu core minimal; implement mapping and IO in the extension.
+* **Reuse existing SDKs:** leverage GraphAr and Arrow C++ SDKs for I/O and schema handling.
+* **Parallel processing:** employ parallel RecordBatch conversion to improve ingestion throughput where applicable.
+* **Conservative scope:** implement robust support for common Arrow types and GraphAr patterns before expanding to edge cases.
+
+---
+
+## Operational guidance
+
+* Use `CALL GRAPHAR_METADATA` to discover table names and column layouts before creating target tables in Kuzu.
+* Ensure target Kuzu table schemas align with GraphAr/Arrow column types; apply explicit conversions in SQL or transform data when necessary.
+* For deterministic exports of edges or id-sensitive payloads, include `internal_id` in the query projection and order by that column.
+* Validate exported YAML and payload files with your downstream GraphAr tooling when integrating exported artifacts.
+* Exports stream query results to the target directory; design downstream validation to tolerate or detect partial outputs when a query or write fails.
+
+---
+
+## Example end-to-end workflow
+
+1. Load the extension: `LOAD EXTENSION "<path>/libgraphar.kuzu_extension";`.
+2. Inspect input metadata: `CALL GRAPHAR_METADATA('<in.graph.yml>')` and identify table names/columns.
+3. Create Kuzu table(s) matching expected types and primary-key semantics.
+4. Ingest data: `COPY <Table> FROM '<in.graph.yml>' (file_format='graphar', table_name='<name>');`.
+5. Validate data in Kuzu with representative queries.
+6. Export data when needed: `COPY (<query>) TO '<out.graph.yml.graphar>' (table_name='<name>', target_dir='<outdir>');`.
+7. Optionally inspect exported YAML files and payloads with GraphAr tooling.
+
+---
+
+## Limitations and planned improvements
+
+Current limitations are acknowledged and prioritized for engineering progress. Planned improvements include:
+
+* Enhanced export metadata generation (more complete YAML with field descriptions and validation metadata).
+* Expanded Arrow/GraphAr type coverage, including nested and complex types.
+* Automated round-trip tests that verify fidelity for representative datasets.
+* Configurable writer backend and tunable parallelism for Arrow writers.
+* Improved diagnostics and error messages for developer workflows.
+
+---
+
+## Files of interest
+
+* `src/extension/` — core implementation for parsing, mapping, and IO.
+* `tests/` — example programs and verification utilities demonstrating metadata inspection, ingestion, query, and export workflows.
+
+---
+
+## Notes
+
+This branch implements a pragmatic, coherent set of GraphAr-related capabilities for Kuzu. It is suitable for experimentation, integration testing, and engineering verification. Users should validate type mappings and metadata semantics against their target GraphAr tooling when producing or consuming exported artifacts.
diff --git a/cpp/extensions/kuzu-extension/src/function/CMakeLists.txt b/cpp/extensions/kuzu-extension/src/function/CMakeLists.txt
new file mode 100644
index 00000000..96d39f35
--- /dev/null
+++ b/cpp/extensions/kuzu-extension/src/function/CMakeLists.txt
@@ -0,0 +1,11 @@
+add_library(kuzu_graphar_function
+ OBJECT
+ graphar_scan.cpp
+ graphar_scan_bindfunc.cpp
+ graphar_export.cpp
+ graphar_export_bindfunc.cpp
+ graphar_metadata.cpp)
+
+set(GRAPHAR_EXTENSION_OBJECT_FILES
+ ${GRAPHAR_EXTENSION_OBJECT_FILES} $<TARGET_OBJECTS:kuzu_graphar_function>
+ PARENT_SCOPE)
diff --git a/cpp/extensions/kuzu-extension/src/function/graphar_export.cpp b/cpp/extensions/kuzu-extension/src/function/graphar_export.cpp
new file mode 100644
index 00000000..52a0d64d
--- /dev/null
+++ b/cpp/extensions/kuzu-extension/src/function/graphar_export.cpp
@@ -0,0 +1,404 @@
+#include "function/graphar_export.h"
+
+using namespace kuzu::function;
+using namespace kuzu::common;
+
+namespace kuzu {
+namespace graphar_extension {
+
+void initSharedState(ExportFuncSharedState& sharedState, main::ClientContext& context,
+ const ExportFuncBindData& bindData) {
+ sharedState.init(context, bindData);
+}
+
+std::shared_ptr<ExportFuncSharedState> createSharedStateFunc() {
+ return std::make_shared<ExportGrapharSharedState>();
+}
+
+std::unique_ptr<ExportFuncLocalState> initLocalState(main::ClientContext&,
+ const ExportFuncBindData&, std::vector<bool>) {
+ return std::make_unique<ExportGrapharLocalState>();
+}
+
+void sinkFunc(ExportFuncSharedState&, ExportFuncLocalState& localState,
+ [[maybe_unused]] const ExportFuncBindData& bindData,
+ std::vector<std::shared_ptr<ValueVector>> inputVectors) {
+ auto& grapharLocalState = localState.cast<ExportGrapharLocalState>();
+
+ // ATTENTION: postpone buffer creation until first sink call, because we can't
+ // know the schema before that (It's hard for bindData to get the type info).
+ if (!grapharLocalState.buffer) {
+ // schema of input vectors
+ std::vector<PropMeta> schema_to_create;
+
+ // fill the schema and create buffer in local state.
+ KU_ASSERT(inputVectors.size() == bindData.columnNames.size());
+ for (size_t i = 0; i < inputVectors.size(); i++) {
+ schema_to_create.push_back(PropMeta{bindData.columnNames[i],
+ kuzuTypeToGrapharType(inputVectors[i]->dataType), Cardinality::SINGLE});
+ }
+
+ grapharLocalState.buffer = std::make_shared<WriteRowsBuffer>(std::move(schema_to_create));
+ }
+
+ auto& buffer = grapharLocalState.buffer;
+ const auto& schema = grapharLocalState.buffer->Schema();
+
+ if (inputVectors.size() != schema.size()) {
+ throw common::RuntimeException("inputVectors size != schema size");
+ }
+
+ // Compute the number of logical rows in the current batch (supporting mixed flat / unflat
+ // vectors) A flat vector is treated as selSize = 1 (broadcasted), while an unflat vector uses
+ // its own selSize.
+ size_t num_rows = 1;
+ for (size_t c = 0; c < inputVectors.size(); ++c) {
+ auto& v = inputVectors[c];
+ if (!v->state->isFlat()) {
+ auto s = static_cast<size_t>(v->state->getSelVector().getSelSize());
+ if (s > num_rows) {
+ num_rows = s;
+ }
+ }
+ }
+
+ // Optional: if prefer a fail-fast policy for inconsistent non-flat columns,
+ // you can disable the “max” strategy above and instead check that all non-flat
+ // columns have the same selSize, throwing an exception otherwise. Example:
+ // size_t expected = 0;
+ // for (...) if (!v->state->isFlat()) { if (expected==0) expected = s; else if (expected != s)
+ // throw ...; }
+ // The current implementation adopts a pad-null strategy to improve robustness
+ // and compatibility.
+
+ for (size_t logicalRow = 0; logicalRow < num_rows; ++logicalRow) {
+ size_t rid = buffer->NewRow();
+
+ for (size_t col = 0; col < schema.size(); ++col) {
+ const auto& meta = schema[col];
+ auto& vecPtr = inputVectors[col];
+
+ // If the vector is flat: always use sel[0].
+ // If unflat: use sel[logicalRow] if logicalRow < selSize,
+ // otherwise treat as missing (null).
+ uint32_t physPos = 0;
+ if (vecPtr->state->isFlat()) {
+ physPos = vecPtr->state->getSelVector()[0];
+ } else {
+ auto selSize = static_cast<size_t>(vecPtr->state->getSelVector().getSelSize());
+ if (logicalRow >= selSize) {
+ // pad-null: this column has no value for the current logicalRow,
+ // leave as monostate (not written).
+ continue;
+ }
+ physPos = vecPtr->state->getSelVector()[logicalRow];
+ }
+
+ // use physical position to check for null
+ if (vecPtr->isNull(physPos)) {
+ continue; // keep as monostate
+ }
+
+ switch (meta.type) {
+ case Type::INT64:
+ case Type::TIMESTAMP:
+ buffer->SetProperty(rid, meta.name, Scalar(vecPtr->getValue<int64_t>(physPos)));
+ break;
+ case Type::INT32:
+ case Type::DATE:
+ buffer->SetProperty(rid, meta.name, Scalar(vecPtr->getValue<int32_t>(physPos)));
+ break;
+ case Type::DOUBLE:
+ buffer->SetProperty(rid, meta.name, Scalar(vecPtr->getValue<double>(physPos)));
+ break;
+ case Type::FLOAT:
+ buffer->SetProperty(rid, meta.name, Scalar(vecPtr->getValue<float>(physPos)));
+ break;
+ case Type::STRING:
+ buffer->SetProperty(rid, meta.name,
+ Scalar(vecPtr->getValue<ku_string_t>(physPos).getAsString()));
+ break;
+ case Type::BOOL:
+ buffer->SetProperty(rid, meta.name, Scalar(vecPtr->getValue<bool>(physPos)));
+ break;
+ default:
+ throw common::RuntimeException(
+ common::stringFormat("Unsupported type for property '{}'", meta.name));
+ }
+ }
+ }
+}
+
+void combineFunc(ExportFuncSharedState& sharedState, ExportFuncLocalState& localState) {
+ auto& grapharSharedState = sharedState.cast<ExportGrapharSharedState>();
+ auto& grapharLocalState = localState.cast<ExportGrapharLocalState>();
+
+ // nothing to combine if local buffer is null, just return.
+ // this can happen if sink haven't been called yet.
+ if (!grapharLocalState.buffer) {
+ return;
+ }
+
+ auto& buffer = grapharLocalState.buffer;
+ auto& schema = grapharLocalState.buffer->Schema();
+
+ // Helper: find index of a property by a list of candidate names (case-sensitive).
+ auto findIndexByCandidates =
+ [&schema](const std::vector<std::string>& candidates) -> std::optional<size_t> {
+ for (const auto& cand : candidates) {
+ for (size_t i = 0; i < schema.size(); ++i) {
+ if (schema[i].name == cand)
+ return i;
+ }
+ }
+ return std::nullopt;
+ };
+
+ // If exporting edges, try to locate src/dst indices once before looping.
+ std::optional<size_t> srcIdxOpt, dstIdxOpt;
+ if (grapharSharedState.is_edge) {
+ srcIdxOpt = findIndexByCandidates(srcCandidates);
+ dstIdxOpt = findIndexByCandidates(dstCandidates);
+ if (!srcIdxOpt || !dstIdxOpt) {
+ // If not found, try a fallback: assume first two columns are src/dst
+ if (schema.size() >= 2) {
+ srcIdxOpt = 0;
+ dstIdxOpt = 1;
+ } else {
+ throw common::RuntimeException{common::stringFormat(
+ "Cannot locate src/dst properties in schema for edge export.")};
+ }
+ }
+ }
+
+ std::lock_guard<std::mutex> lck{grapharSharedState.mtx};
+
+ // Iterate over rows and convert each row into a Vertex or Edge.
+ auto& rows = buffer->GetRows();
+ for (auto& row : rows) {
+ if (!grapharSharedState.is_edge) {
+ // Vertex export
+ graphar::builder::Vertex vertex;
+
+ // For each property in schema, if it's set, push to vertex.
+ for (size_t i = 0; i < schema.size(); ++i) {
+ const auto& meta = schema[i];
+ const auto& cell = row.props[i];
+
+ // Skip unset properties.
+ if (std::holds_alternative<std::monostate>(cell))
+ continue;
+
+ if (meta.card == Cardinality::SINGLE) {
+ // Single-valued property
+ const Scalar& s = std::get<Scalar>(cell);
+ std::visit(
+ [&](auto&& val) {
+ using T = std::decay_t<decltype(val)>;
+ if constexpr (std::is_same_v<T, int64_t>) {
+ vertex.AddProperty(meta.name, val);
+ } else if constexpr (std::is_same_v<T, int32_t>) {
+ vertex.AddProperty(meta.name, val);
+ } else if constexpr (std::is_same_v<T, double>) {
+ vertex.AddProperty(meta.name, val);
+ } else if constexpr (std::is_same_v<T, float>) {
+ vertex.AddProperty(meta.name, val);
+ } else if constexpr (std::is_same_v<T, std::string>) {
+ vertex.AddProperty(meta.name, val);
+ } else if constexpr (std::is_same_v<T, bool>) {
+ vertex.AddProperty(meta.name, val);
+ } else {
+ // unreachable
+ }
+ },
+ s);
+ } else {
+ throw common::NotImplementedException{
+ "GraphAr edge multi-valued properties are not implemented."};
+ }
+ }
+
+ // Add constructed vertex to verticesBuilder and check status.
+ if (!grapharSharedState.verticesBuilder) {
+ throw common::RuntimeException{"verticesBuilder is null."};
+ }
+ auto st_v = grapharSharedState.verticesBuilder->AddVertex(vertex);
+ if (!st_v.ok()) {
+ throw common::RuntimeException{
+ common::stringFormat("AddVertex failed: {}", st_v.message())};
+ }
+
+ } else {
+ // Edge export
+ // retrieve src/dst indices (guaranteed present above)
+ size_t srcIdx = *srcIdxOpt;
+ size_t dstIdx = *dstIdxOpt;
+
+ // src/dst must be set and scalar
+ const auto& srcCell = row.props[srcIdx];
+ const auto& dstCell = row.props[dstIdx];
+ if (std::holds_alternative<std::monostate>(srcCell) ||
+ std::holds_alternative<std::monostate>(dstCell)) {
+ throw common::RuntimeException{"edge row missing src or dst value."};
+ }
+ if (!std::holds_alternative<Scalar>(srcCell) ||
+ !std::holds_alternative<Scalar>(dstCell)) {
+ throw common::RuntimeException{"edge src/dst must be scalar."};
+ }
+
+ // Extract src/dst as integer (prefer int64_t, allow int32_t)
+ int64_t srcId = 0, dstId = 0;
+ const Scalar& ssrc = std::get<Scalar>(srcCell);
+ const Scalar& sdst = std::get<Scalar>(dstCell);
+ // visitor for extracting integer id
+ auto extractId = [](const Scalar& sc) -> int64_t {
+ if (std::holds_alternative<int64_t>(sc))
+ return std::get<int64_t>(sc);
+ if (std::holds_alternative<int32_t>(sc))
+ return static_cast<int64_t>(std::get<int32_t>(sc));
+ // also allow strings that look like integers? Not by default: throw
+ throw common::RuntimeException{"edge endpoint is not integer type."};
+ };
+ srcId = extractId(ssrc);
+ dstId = extractId(sdst);
+
+ // construct edge
+ graphar::builder::Edge edge(srcId, dstId);
+
+ // add remaining properties except src/dst themselves
+ for (size_t i = 0; i < schema.size(); ++i) {
+ if (i == srcIdx || i == dstIdx)
+ continue; // skip endpoints
+ const auto& meta = schema[i];
+ const auto& cell = row.props[i];
+ if (std::holds_alternative<std::monostate>(cell))
+ continue;
+
+ if (meta.card == Cardinality::SINGLE) {
+ const Scalar& s = std::get<Scalar>(cell);
+ std::visit(
+ [&](auto&& val) {
+ using T = std::decay_t<decltype(val)>;
+ if constexpr (std::is_same_v<T, int64_t>) {
+ edge.AddProperty(meta.name, val);
+ } else if constexpr (std::is_same_v<T, int32_t>) {
+ edge.AddProperty(meta.name, val);
+ } else if constexpr (std::is_same_v<T, double>) {
+ edge.AddProperty(meta.name, val);
+ } else if constexpr (std::is_same_v<T, float>) {
+ edge.AddProperty(meta.name, val);
+ } else if constexpr (std::is_same_v<T, std::string>) {
+ edge.AddProperty(meta.name, val);
+ } else if constexpr (std::is_same_v<T, bool>) {
+ edge.AddProperty(meta.name, val);
+ }
+ },
+ s);
+ } else {
+ throw common::NotImplementedException{
+ "GraphAr edge multi-valued properties are not implemented."};
+ }
+ }
+
+ // add edge to edgesBuilder and check status
+ if (!grapharSharedState.edgesBuilder) {
+ throw common::RuntimeException{"edgesBuilder is null."};
+ }
+ auto st_e = grapharSharedState.edgesBuilder->AddEdge(edge);
+ if (!st_e.ok()) {
+ throw common::RuntimeException{
+ common::stringFormat("AddEdge failed: {}", st_e.message())};
+ }
+ }
+ }
+}
+
+void finalizeFunc(ExportFuncSharedState& sharedState) {
+ auto& grapharSharedState = sharedState.cast<ExportGrapharSharedState>();
+
+ if (!grapharSharedState.is_edge) {
+ if (!grapharSharedState.verticesBuilder) {
+ throw common::RuntimeException{"verticesBuilder is null in finalize."};
+ }
+ auto st = grapharSharedState.verticesBuilder->Dump();
+ if (!st.ok()) {
+ throw common::RuntimeException{
+ common::stringFormat("VerticesBuilder Finalize failed: {}", st.message())};
+ }
+ grapharSharedState.verticesBuilder->Clear();
+ } else {
+ if (!grapharSharedState.edgesBuilder) {
+ throw common::RuntimeException{"edgesBuilder is null in finalize."};
+ }
+ auto st = grapharSharedState.edgesBuilder->Dump();
+ if (!st.ok()) {
+ throw common::RuntimeException{
+ common::stringFormat("EdgesBuilder Finalize failed: {}", st.message())};
+ }
+ grapharSharedState.edgesBuilder->Clear();
+ }
+}
+
+void ExportGrapharSharedState::init([[maybe_unused]] main::ClientContext& context,
+ const ExportFuncBindData& bindData) {
+ const ExportGrapharBindData& grapharBindData = bindData.constCast<ExportGrapharBindData>();
+ std::shared_ptr<GraphInfo> graph_info = grapharBindData.graphInfo;
+ GrapharExportOptions exportOptions = grapharBindData.exportOptions;
+ std::string tableName = grapharBindData.tableName;
+ std::string targetDir = grapharBindData.targetDir;
+ ValidateLevel validateLevel = grapharBindData.validateLevel;
+
+ auto vertex_infos = graph_info->GetVertexInfos();
+ auto edge_infos = graph_info->GetEdgeInfos();
+
+ // Try vertex first
+ for (const auto& v_info : vertex_infos) {
+ if (v_info->GetType() == tableName) {
+ vertexInfo = v_info;
+ verticesBuilder = std::make_shared<builder::VerticesBuilder>(vertexInfo, targetDir, 0L,
+ exportOptions.wopt, validateLevel);
+ is_edge = false;
+ return;
+ }
+ }
+
+ // Try edge if not vertex
+ for (const auto& e_info : edge_infos) {
+ std::string src_type = e_info->GetSrcType();
+ std::string edge_type = e_info->GetEdgeType();
+ std::string dst_type = e_info->GetDstType();
+ std::string full_edge_name =
+ src_type + REGULAR_SEPARATOR + edge_type + REGULAR_SEPARATOR + dst_type;
+
+ // Get src type count
+ auto maybe_from_vertices_collection =
+ graphar::VerticesCollection::Make(graph_info, src_type);
+ KU_ASSERT(!maybe_from_vertices_collection.has_error());
+ int vertexNum = maybe_from_vertices_collection.value()->size();
+
+ if (full_edge_name == tableName) {
+ edgeInfo = e_info;
+ edgesBuilder = std::make_shared<builder::EdgesBuilder>(edgeInfo, targetDir,
+ AdjListType::ordered_by_source, vertexNum, exportOptions.wopt, validateLevel);
+ is_edge = true;
+ return;
+ }
+ }
+}
+
+function_set GrapharExportFunction::getFunctionSet() {
+ function_set functionSet;
+ auto exportFunc = std::make_unique<ExportFunction>(name);
+ exportFunc->bind = bindFunc;
+ exportFunc->initLocalState = initLocalState;
+ exportFunc->createSharedState = createSharedStateFunc;
+ exportFunc->initSharedState = initSharedState;
+ exportFunc->sink = sinkFunc;
+ exportFunc->combine = combineFunc;
+ exportFunc->finalize = finalizeFunc;
+ functionSet.push_back(std::move(exportFunc));
+ return functionSet;
+}
+
+} // namespace graphar_extension
+} // namespace kuzu
diff --git a/cpp/extensions/kuzu-extension/src/function/graphar_export_bindfunc.cpp b/cpp/extensions/kuzu-extension/src/function/graphar_export_bindfunc.cpp
new file mode 100644
index 00000000..46071d7b
--- /dev/null
+++ b/cpp/extensions/kuzu-extension/src/function/graphar_export_bindfunc.cpp
@@ -0,0 +1,503 @@
+#include "function/graphar_export.h"
+
+namespace kuzu {
+namespace graphar_extension {
+
+using namespace function;
+using namespace common;
+
+void GrapharExportOptions::setCSVNullString(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::STRING) {
+ throw common::RuntimeException{
+ common::stringFormat("CSV null string option expects a string value, got: {}.",
+ value.getDataType().toString())};
+ }
+ csvOptionBuilder->null_string(value.getValue<std::string>());
+}
+
+void GrapharExportOptions::setCSVDelimiter(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::STRING) {
+ throw common::RuntimeException{
+ common::stringFormat("CSV delimiter option expects a string value, got: {}.",
+ value.getDataType().toString())};
+ }
+ auto strVal = value.getValue<std::string>();
+ if (strVal.length() != 1) {
+ throw common::RuntimeException{common::stringFormat(
+ "CSV delimiter option expects a single character, got: {}.", strVal)};
+ }
+ csvOptionBuilder->delimiter(strVal[0]);
+}
+
+void GrapharExportOptions::setCSVIncludeHeader(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::BOOL) {
+ throw common::RuntimeException{
+ common::stringFormat("CSV include header option expects a boolean value, got: {}.",
+ value.getDataType().toString())};
+ }
+ csvOptionBuilder->include_header(value.getValue<bool>());
+}
+
+void GrapharExportOptions::setCSVBatchSize(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::INT64) {
+ throw common::RuntimeException{
+ common::stringFormat("CSV batch size option expects an integer value, got: {}.",
+ value.getDataType().toString())};
+ }
+ auto intVal = value.getValue<int64_t>();
+ if (intVal <= 0) {
+ throw common::RuntimeException{common::stringFormat(
+ "CSV batch size option expects a positive integer value, got: {}.", intVal)};
+ }
+ csvOptionBuilder->batch_size(static_cast<int32_t>(intVal));
+}
+
+void GrapharExportOptions::setCSVEOL(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::STRING) {
+ throw common::RuntimeException{common::stringFormat(
+ "CSV EOL option expects a string value, got: {}.", value.getDataType().toString())};
+ }
+ csvOptionBuilder->eol(value.getValue<std::string>());
+}
+
+void GrapharExportOptions::setCSVQuotingStyle(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::STRING) {
+ throw common::RuntimeException{
+ common::stringFormat("CSV quoting style option expects a string value, got: {}.",
+ value.getDataType().toString())};
+ }
+ auto strVal = common::StringUtils::getUpper(value.getValue<std::string>());
+ if (strVal == "ALLVALID") {
+ csvOptionBuilder->quoting_style(arrow::csv::QuotingStyle::AllValid);
+ } else if (strVal == "NONE") {
+ csvOptionBuilder->quoting_style(arrow::csv::QuotingStyle::None);
+ } else if (strVal == "NEEDED") {
+ csvOptionBuilder->quoting_style(arrow::csv::QuotingStyle::Needed);
+ } else {
+ throw common::RuntimeException{
+ common::stringFormat("Unrecognized csv quoting style option: {}.", value.toString())};
+ }
+}
+
+void GrapharExportOptions::setParquetDictionaryPageSizeLimit(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::INT64) {
+ throw common::RuntimeException{common::stringFormat(
+ "Parquet dictionary pagesize limit option expects an integer value, got: {}.",
+ value.getDataType().toString())};
+ }
+ auto intVal = value.getValue<int64_t>();
+ if (intVal <= 0) {
+ throw common::RuntimeException{
+ common::stringFormat("Parquet dictionary pagesize limit option expects a positive "
+ "integer value, got: {}.",
+ intVal)};
+ }
+ parquetOptionBuilder->dictionary_pagesize_limit(intVal);
+}
+
+void GrapharExportOptions::setParquetWriteBatchSize(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::INT64) {
+ throw common::RuntimeException{common::stringFormat(
+ "Parquet write batch size option expects an integer value, got: {}.",
+ value.getDataType().toString())};
+ }
+ auto intVal = value.getValue<int64_t>();
+ if (intVal <= 0) {
+ throw common::RuntimeException{common::stringFormat(
+ "Parquet write batch size option expects a positive integer value, got: {}.", intVal)};
+ }
+ parquetOptionBuilder->write_batch_size(intVal);
+}
+
+void GrapharExportOptions::setParquetMaxRowGroupLength(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::INT64) {
+ throw common::RuntimeException{common::stringFormat(
+ "Parquet max row group length option expects an integer value, got: {}.",
+ value.getDataType().toString())};
+ }
+ auto intVal = value.getValue<int64_t>();
+ if (intVal <= 0) {
+ throw common::RuntimeException{common::stringFormat(
+ "Parquet max row group length option expects a positive integer value, got: {}.",
+ intVal)};
+ }
+ parquetOptionBuilder->max_row_group_length(intVal);
+}
+
+void GrapharExportOptions::setParquetDataPageSize(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::INT64) {
+ throw common::RuntimeException{
+ common::stringFormat("Parquet data pagesize option expects an integer value, got: {}.",
+ value.getDataType().toString())};
+ }
+ auto intVal = value.getValue<int64_t>();
+ if (intVal <= 0) {
+ throw common::RuntimeException{common::stringFormat(
+ "Parquet data pagesize option expects a positive integer value, got: {}.", intVal)};
+ }
+ parquetOptionBuilder->data_pagesize(intVal);
+}
+
+void GrapharExportOptions::setParquetMaxStatisticsSize(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::INT64) {
+ throw common::RuntimeException{common::stringFormat(
+ "Parquet max statistics size option expects an integer value, got: {}.",
+ value.getDataType().toString())};
+ }
+ auto intVal = value.getValue<int64_t>();
+ if (intVal <= 0) {
+ throw common::RuntimeException{common::stringFormat(
+ "Parquet max statistics size option expects a positive integer value, got: {}.",
+ intVal)};
+ }
+ parquetOptionBuilder->max_statistics_size(static_cast<size_t>(intVal));
+}
+
+void GrapharExportOptions::setParquetCompressionLevel(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::INT64) {
+ throw common::RuntimeException{common::stringFormat(
+ "Parquet compression level option expects an integer value, got: {}.",
+ value.getDataType().toString())};
+ }
+ parquetOptionBuilder->compression_level(static_cast<int>(value.getValue<int64_t>()));
+}
+
+void GrapharExportOptions::setParquetDataPageVersion(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::STRING) {
+ throw common::RuntimeException{common::stringFormat(
+ "Parquet data page version option expects a string value, got: {}.",
+ value.getDataType().toString())};
+ }
+ auto strVal = common::StringUtils::getUpper(value.getValue<std::string>());
+ if (strVal == "V1") {
+ parquetOptionBuilder->data_page_version(::parquet::ParquetDataPageVersion::V1);
+ } else if (strVal == "V2") {
+ parquetOptionBuilder->data_page_version(::parquet::ParquetDataPageVersion::V2);
+ } else {
+ throw common::RuntimeException{common::stringFormat(
+ "Unrecognized parquet data page version option: {}.", value.toString())};
+ }
+}
+
+void GrapharExportOptions::setParquetVersion(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::STRING) {
+ throw common::RuntimeException{
+ common::stringFormat("Parquet version option expects a string value, got: {}.",
+ value.getDataType().toString())};
+ }
+ auto strVal = common::StringUtils::getUpper(value.getValue<std::string>());
+ if (strVal == "PARQUET_1_0") {
+ parquetOptionBuilder->version(::parquet::ParquetVersion::PARQUET_1_0);
+ } else if (strVal == "PARQUET_2_0") {
+ parquetOptionBuilder->version(::parquet::ParquetVersion::PARQUET_2_0);
+ } else if (strVal == "PARQUET_2_4") {
+ parquetOptionBuilder->version(::parquet::ParquetVersion::PARQUET_2_4);
+ } else if (strVal == "PARQUET_2_6") {
+ parquetOptionBuilder->version(::parquet::ParquetVersion::PARQUET_2_6);
+ } else if (strVal == "PARQUET_2_LATEST") {
+ parquetOptionBuilder->version(::parquet::ParquetVersion::PARQUET_2_LATEST);
+ } else {
+ throw common::RuntimeException{
+ common::stringFormat("Unrecognized parquet version option: {}.", value.toString())};
+ }
+}
+
+void GrapharExportOptions::setParquetEncoding(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::STRING) {
+ throw common::RuntimeException{
+ common::stringFormat("Parquet encoding option expects a string value, got: {}.",
+ value.getDataType().toString())};
+ }
+ auto strVal = common::StringUtils::getUpper(value.getValue<std::string>());
+ if (strVal == "PLAIN") {
+ parquetOptionBuilder->encoding(::parquet::Encoding::PLAIN);
+ } else if (strVal == "RLE") {
+ parquetOptionBuilder->encoding(::parquet::Encoding::RLE);
+ } else if (strVal == "BIT_PACKED") {
+ parquetOptionBuilder->encoding(::parquet::Encoding::BIT_PACKED);
+ } else if (strVal == "DELTA_BINARY_PACKED") {
+ parquetOptionBuilder->encoding(::parquet::Encoding::DELTA_BINARY_PACKED);
+ } else if (strVal == "DELTA_LENGTH_BYTE_ARRAY") {
+ parquetOptionBuilder->encoding(::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY);
+ } else if (strVal == "DELTA_BYTE_ARRAY") {
+ parquetOptionBuilder->encoding(::parquet::Encoding::DELTA_BYTE_ARRAY);
+ } else if (strVal == "RLE_DICTIONARY") {
+ parquetOptionBuilder->encoding(::parquet::Encoding::RLE_DICTIONARY);
+ } else if (strVal == "BYTE_STREAM_SPLIT") {
+ parquetOptionBuilder->encoding(::parquet::Encoding::BYTE_STREAM_SPLIT);
+ } else {
+ throw common::RuntimeException{
+ common::stringFormat("Unrecognized parquet encoding option: {}.", value.toString())};
+ }
+}
+
+void GrapharExportOptions::setParquetCompression(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::STRING) {
+ throw common::RuntimeException{
+ common::stringFormat("Parquet compression option expects a string value, got: {}.",
+ value.getDataType().toString())};
+ }
+ auto strVal = common::StringUtils::getUpper(value.getValue<std::string>());
+ if (strVal == "UNCOMPRESSED") {
+ parquetOptionBuilder->compression(arrow::Compression::UNCOMPRESSED);
+ } else if (strVal == "SNAPPY") {
+ parquetOptionBuilder->compression(arrow::Compression::SNAPPY);
+ } else if (strVal == "GZIP") {
+ parquetOptionBuilder->compression(arrow::Compression::GZIP);
+ } else if (strVal == "BROTLI") {
+ parquetOptionBuilder->compression(arrow::Compression::BROTLI);
+ } else if (strVal == "LZ4") {
+ parquetOptionBuilder->compression(arrow::Compression::LZ4);
+ } else if (strVal == "ZSTD") {
+ parquetOptionBuilder->compression(arrow::Compression::ZSTD);
+ } else {
+ throw common::RuntimeException{
+ common::stringFormat("Unrecognized parquet compression option: {}.", value.toString())};
+ }
+}
+
+void GrapharExportOptions::setParquetCoerceTimestamps(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::STRING) {
+ throw common::RuntimeException{common::stringFormat(
+ "Parquet coerce timestamps option expects a string value, got: {}.",
+ value.getDataType().toString())};
+ }
+ auto strVal = common::StringUtils::getUpper(value.getValue<std::string>());
+ if (strVal == "SECOND") {
+ parquetOptionBuilder->coerce_timestamps(::arrow::TimeUnit::SECOND);
+ } else if (strVal == "MILLI") {
+ parquetOptionBuilder->coerce_timestamps(::arrow::TimeUnit::MILLI);
+ } else if (strVal == "MICRO") {
+ parquetOptionBuilder->coerce_timestamps(::arrow::TimeUnit::MICRO);
+ } else if (strVal == "NANO") {
+ parquetOptionBuilder->coerce_timestamps(::arrow::TimeUnit::NANO);
+ } else {
+ throw common::RuntimeException{common::stringFormat(
+ "Unrecognized parquet coerce timestamps option: {}.", value.toString())};
+ }
+}
+
+void GrapharExportOptions::setParquetEnableDictionary(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::BOOL) {
+ throw common::RuntimeException{common::stringFormat(
+ "Parquet enable dictionary option expects a boolean value, got: {}.",
+ value.getDataType().toString())};
+ }
+ parquetOptionBuilder->enable_dictionary(value.getValue<bool>());
+}
+
+void GrapharExportOptions::setParquetEnableStatistics(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::BOOL) {
+ throw common::RuntimeException{common::stringFormat(
+ "Parquet enable statistics option expects a boolean value, got: {}.",
+ value.getDataType().toString())};
+ }
+ parquetOptionBuilder->enable_statistics(value.getValue<bool>());
+}
+
+void GrapharExportOptions::setParquetEnableStoreDecimalAsInteger(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::BOOL) {
+ throw common::RuntimeException{common::stringFormat(
+ "Parquet enable store decimal as integer option expects a boolean value, got: {}.",
+ value.getDataType().toString())};
+ }
+ parquetOptionBuilder->enable_store_decimal_as_integer(value.getValue<bool>());
+}
+
+void GrapharExportOptions::setParquetEnableWritePageIndex(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::BOOL) {
+ throw common::RuntimeException{common::stringFormat(
+ "Parquet enable write page index option expects a boolean value, got: {}.",
+ value.getDataType().toString())};
+ }
+ parquetOptionBuilder->enable_write_page_index(value.getValue<bool>());
+}
+
+void GrapharExportOptions::setParquetCompliantNestedTypes(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::BOOL) {
+ throw common::RuntimeException{common::stringFormat(
+ "Parquet compliant nested types option expects a boolean value, got: {}.",
+ value.getDataType().toString())};
+ }
+ parquetOptionBuilder->compliant_nested_types(value.getValue<bool>());
+}
+
+void GrapharExportOptions::setParquetUseThreads(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::BOOL) {
+ throw common::RuntimeException{
+ common::stringFormat("Parquet use threads option expects a boolean value, got: {}.",
+ value.getDataType().toString())};
+ }
+ parquetOptionBuilder->use_threads(value.getValue<bool>());
+}
+
+void GrapharExportOptions::setParquetEnableDeprecatedInt96Timestamps(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::BOOL) {
+ throw common::RuntimeException{common::stringFormat(
+ "Parquet enable deprecated int96 timestamps option expects a boolean value, got: {}.",
+ value.getDataType().toString())};
+ }
+ parquetOptionBuilder->enable_deprecated_int96_timestamps(value.getValue<bool>());
+}
+
+void GrapharExportOptions::setParquetAllowTruncatedTimestamps(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::BOOL) {
+ throw common::RuntimeException{common::stringFormat(
+ "Parquet allow truncated timestamps option expects a boolean value, got: {}.",
+ value.getDataType().toString())};
+ }
+ parquetOptionBuilder->allow_truncated_timestamps(value.getValue<bool>());
+}
+
+void GrapharExportOptions::setParquetStoreSchema(common::Value& value) {
+ if (value.getDataType().getLogicalTypeID() != LogicalTypeID::BOOL) {
+ throw common::RuntimeException{
+ common::stringFormat("Parquet store schema option expects a boolean value, got: {}.",
+ value.getDataType().toString())};
+ }
+ parquetOptionBuilder->store_schema(value.getValue<bool>());
+}
+
+GrapharExportOptions::GrapharExportOptions(case_insensitive_map_t<common::Value> parsingOptions) {
+ wopt = std::make_shared<WriterOptions>();
+
+ // get CSV/Parquet/ORC writer options
+ csvOptionBuilder = std::make_shared<WriterOptions::CSVOptionBuilder>(wopt);
+ parquetOptionBuilder = std::make_shared<WriterOptions::ParquetOptionBuilder>(wopt);
+ orcOptionBuilder = std::make_shared<WriterOptions::ORCOptionBuilder>(wopt);
+
+ for (auto& [name, value] : parsingOptions) {
+ if (name == "CSV-NULL_STRING") {
+ setCSVNullString(value);
+ } else if (name == "CSV-DELIMITER") {
+ setCSVDelimiter(value);
+ } else if (name == "CSV-INCLUDE_HEADER") {
+ setCSVIncludeHeader(value);
+ } else if (name == "CSV-BATCH_SIZE") {
+ setCSVBatchSize(value);
+ } else if (name == "CSV-EOL") {
+ setCSVEOL(value);
+ } else if (name == "CSV-QUOTING_STYLE") {
+ setCSVQuotingStyle(value);
+ } else if (name == "PARQUET-DICTIONARY_PAGESIZE_LIMIT") {
+ setParquetDictionaryPageSizeLimit(value);
+ } else if (name == "PARQUET-WRITE_BATCH_SIZE") {
+ setParquetWriteBatchSize(value);
+ } else if (name == "PARQUET-MAX_ROW_GROUP_LENGTH") {
+ setParquetMaxRowGroupLength(value);
+ } else if (name == "PARQUET-DATA_PAGESIZE") {
+ setParquetDataPageSize(value);
+ } else if (name == "PARQUET-MAX_STATISTICS_SIZE") {
+ setParquetMaxStatisticsSize(value);
+ } else if (name == "PARQUET-COMPRESSION_LEVEL") {
+ setParquetCompressionLevel(value);
+ } else if (name == "PARQUET-DATA_PAGE_VERSION") {
+ setParquetDataPageVersion(value);
+ } else if (name == "PARQUET-VERSION") {
+ setParquetVersion(value);
+ } else if (name == "PARQUET-ENCODING") {
+ setParquetEncoding(value);
+ } else if (name == "PARQUET-COMPRESSION") {
+ setParquetCompression(value);
+ } else if (name == "PARQUET-COERCE_TIMESTAMPS") {
+ setParquetCoerceTimestamps(value);
+ } else if (name == "PARQUET-ENABLE_DICTIONARY") {
+ setParquetEnableDictionary(value);
+ } else if (name == "PARQUET-ENABLE_STATISTICS") {
+ setParquetEnableStatistics(value);
+ } else if (name == "PARQUET-ENABLE_STORE_DECIMAL_AS_INTEGER") {
+ setParquetEnableStoreDecimalAsInteger(value);
+ } else if (name == "PARQUET-ENABLE_WRITE_PAGE_INDEX") {
+ setParquetEnableWritePageIndex(value);
+ } else if (name == "PARQUET-COMPLIANT_NESTED_TYPES") {
+ setParquetCompliantNestedTypes(value);
+ } else if (name == "PARQUET-USE_THREADS") {
+ setParquetUseThreads(value);
+ } else if (name == "PARQUET-ENABLE_DEPRECATED_INT96_TIMESTAMPS") {
+ setParquetEnableDeprecatedInt96Timestamps(value);
+ } else if (name == "PARQUET-ALLOW_TRUNCATED_TIMESTAMPS") {
+ setParquetAllowTruncatedTimestamps(value);
+ } else if (name == "PARQUET-STORE_SCHEMA") {
+ setParquetStoreSchema(value);
+ }
+ }
+
+ csvOptionBuilder->build();
+ parquetOptionBuilder->build();
+ orcOptionBuilder->build();
+}
+
+ExportGrapharBindData::ExportGrapharBindData(std::vector<std::string> columnNames,
+ const std::string& fileName, GrapharExportOptions grapharExportOptions, std::string tableName,
+ std::string targetDir, ValidateLevel validateLevel)
+ : ExportFuncBindData(std::move(columnNames), std::move(fileName)),
+ exportOptions(std::move(grapharExportOptions)), tableName(std::move(tableName)),
+ targetDir(std::move(targetDir)), validateLevel(validateLevel) {
+ auto absolute_path = this->fileName;
+ // Load graph info from the file path
+ graphInfo = graphar::GraphInfo::Load(absolute_path).value();
+ if (!graphInfo) {
+ throw BinderException("GraphAr's GraphInfo could not be loaded from " + absolute_path);
+ }
+}
+
+std::unique_ptr<ExportFuncBindData> bindFunc(ExportFuncBindInput& bindInput) {
+ GrapharExportOptions grapharExportOptions{bindInput.parsingOptions};
+ // get table name.
+ auto table_it = bindInput.parsingOptions.find("TABLE_NAME");
+ if (table_it != bindInput.parsingOptions.end()) {
+ if (table_it->second.getDataType().getLogicalTypeID() != LogicalTypeID::STRING) {
+ throw common::RuntimeException{
+ common::stringFormat("Table name option expects a string value, got: {}.",
+ table_it->second.getDataType().toString())};
+ }
+ } else {
+ throw BinderException("Table name must be specified in the parsing options with key "
+ "'TABLE_NAME' for GraphAr export.");
+ }
+ std::string tableName = table_it->second.getValue<std::string>();
+
+ // get target directory.
+ std::string targetDir = DEFAULT_TARGET_DIR;
+ auto dir_it = bindInput.parsingOptions.find("TARGET_DIR");
+ if (dir_it != bindInput.parsingOptions.end()) {
+ if (dir_it->second.getDataType().getLogicalTypeID() != LogicalTypeID::STRING) {
+ throw common::RuntimeException{
+ common::stringFormat("Target directory option expects a string value, got: {}.",
+ dir_it->second.getDataType().toString())};
+ }
+ targetDir = dir_it->second.getValue<std::string>();
+ }
+
+ // get validate level.
+ ValidateLevel validateLevel = ValidateLevel::strong_validate;
+ auto validate_it = bindInput.parsingOptions.find("VALIDATE_LEVEL");
+ if (validate_it != bindInput.parsingOptions.end()) {
+ if (validate_it->second.getDataType().getLogicalTypeID() != LogicalTypeID::STRING) {
+ throw common::RuntimeException{
+ common::stringFormat("Validate level option expects a string value, got: {}.",
+ validate_it->second.getDataType().toString())};
+ }
+ auto strVal = common::StringUtils::getUpper(validate_it->second.getValue<std::string>());
+ if (strVal == "DEFAULT") {
+ validateLevel = ValidateLevel::default_validate;
+ } else if (strVal == "NO") {
+ validateLevel = ValidateLevel::no_validate;
+ } else if (strVal == "WEAK") {
+ validateLevel = ValidateLevel::weak_validate;
+ } else if (strVal == "STRONG") {
+ validateLevel = ValidateLevel::strong_validate;
+ } else {
+ throw common::RuntimeException{common::stringFormat(
+ "Unrecognized validate level option: {}.", validate_it->second.toString())};
+ }
+ }
+
+ // convert '.../ldbc_sample.graph.yml.graphar' to '.../ldbc_sample.graph.yml'.
+ getYamlNameWithoutGrapharLabelInPlace(bindInput.filePath);
+ return std::make_unique<ExportGrapharBindData>(bindInput.columnNames, bindInput.filePath,
+ grapharExportOptions, tableName, targetDir, validateLevel);
+}
+
+} // namespace graphar_extension
+} // namespace kuzu
\ No newline at end of file
diff --git a/cpp/extensions/kuzu-extension/src/function/graphar_metadata.cpp b/cpp/extensions/kuzu-extension/src/function/graphar_metadata.cpp
new file mode 100644
index 00000000..d35602de
--- /dev/null
+++ b/cpp/extensions/kuzu-extension/src/function/graphar_metadata.cpp
@@ -0,0 +1,417 @@
+#include "function/graphar_metadata.h"
+
+using namespace kuzu::function;
+using namespace kuzu::common;
+
+namespace kuzu {
+namespace graphar_extension {
+
+std::unique_ptr<TableFuncBindData> metadataBindFunc(main::ClientContext* context,
+ const TableFuncBindInput* input) {
+ std::string absolute_path = input->getLiteralVal<std::string>(0);
+
+ if (absolute_path.empty()) {
+ throw BinderException("GraphAr metadata requires a valid graphInfo file path.");
+ }
+
+ auto optional_params = input->optionalParams;
+ if (optional_params.find("vertex_type") != optional_params.end() &&
+ optional_params.find("edge_type") != optional_params.end()) {
+ throw BinderException("GraphAr metadata Cannot specify both vertex_type and edge_type.");
+ }
+
+ // Load graph info from the file path
+ auto graph_info = graphar::GraphInfo::Load(absolute_path).value();
+ if (!graph_info) {
+ throw BinderException("GraphAr's GraphInfo could not be loaded from " + absolute_path);
+ }
+ auto vertex_infos = graph_info->GetVertexInfos();
+ auto edge_infos = graph_info->GetEdgeInfos();
+ std::shared_ptr<graphar::VertexInfo> vertex_info = nullptr;
+ std::shared_ptr<graphar::EdgeInfo> edge_info = nullptr;
+
+ GrapharMetadataType metadata_type;
+ std::string table_name;
+ if (optional_params.find("vertex_type") != optional_params.end()) {
+ metadata_type = GrapharMetadataType::VERTEX;
+ table_name = optional_params.at("vertex_type").strVal;
+ for (const auto& v_info : vertex_infos) {
+ if (v_info->GetType() == table_name) {
+ vertex_info = v_info;
+ goto METADATA_TYPE_DEDUCED;
+ }
+ }
+ throw BinderException("Vertex type " + table_name + " not found in the GraphAr graph.");
+ } else if (optional_params.find("edge_type") != optional_params.end()) {
+ metadata_type = GrapharMetadataType::EDGE;
+ table_name = optional_params.at("edge_type").strVal;
+ for (const auto& e_info : edge_infos) {
+ std::string src_type = e_info->GetSrcType();
+ std::string edge_type = e_info->GetEdgeType();
+ std::string dst_type = e_info->GetDstType();
+ std::string full_edge_name =
+ src_type + REGULAR_SEPARATOR + edge_type + REGULAR_SEPARATOR + dst_type;
+ if (full_edge_name == table_name) {
+ edge_info = e_info;
+ goto METADATA_TYPE_DEDUCED;
+ }
+ }
+ throw BinderException("Edge type " + table_name + " not found in the GraphAr graph.");
+ } else {
+ metadata_type = GrapharMetadataType::GRAPH;
+ }
+
+METADATA_TYPE_DEDUCED:
+ std::vector<LogicalType> column_types;
+ std::vector<std::string> column_names;
+
+ // Wanted metadata of the graph
+ switch (metadata_type) {
+ case GrapharMetadataType::GRAPH: {
+ column_names = {
+ "graph_name",
+ "vertex_type_num",
+ "edge_types_num",
+ "vertex_types",
+ "edge_types",
+ "prefix",
+ "version"
+ };
+ column_types.reserve(GRAPH_METADATA_COLUMN_NUM);
+ column_types.emplace_back(LogicalType::STRING());
+ column_types.emplace_back(LogicalType::INT32());
+ column_types.emplace_back(LogicalType::INT32());
+ column_types.emplace_back(LogicalType::LIST(LogicalType::STRING()));
+ column_types.emplace_back(LogicalType::LIST(LogicalType::STRING()));
+ column_types.emplace_back(LogicalType::STRING());
+ column_types.emplace_back(LogicalType::STRING());
+ break;
+ }
+ case GrapharMetadataType::VERTEX: {
+ column_names = {
+ "vertex_type",
+ "vertex_count",
+ "property_names|types|is_primary",
+ "chunk_size",
+ "prefix",
+ "version"
+ };
+ column_types.reserve(VERTEX_METADATA_COLUMN_NUM);
+ column_types.emplace_back(LogicalType::STRING());
+ column_types.emplace_back(LogicalType::INT64());
+ column_types.emplace_back(LogicalType::LIST(LogicalType::STRING()));
+ column_types.emplace_back(LogicalType::INT64());
+ column_types.emplace_back(LogicalType::STRING());
+ column_types.emplace_back(LogicalType::STRING());
+ break;
+ }
+ case GrapharMetadataType::EDGE: {
+ column_names = {
+ "edge_type",
+ "src_type",
+ "dst_type",
+ "directed",
+ "edge_count",
+ "property_names|types|is_primary",
+ "chunk_size",
+ "src_chunk_size",
+ "dst_chunk_size",
+ "prefix",
+ "version"
+ };
+ column_types.reserve(EDGE_METADATA_COLUMN_NUM);
+ column_types.emplace_back(LogicalType::STRING());
+ column_types.emplace_back(LogicalType::STRING());
+ column_types.emplace_back(LogicalType::STRING());
+ column_types.emplace_back(LogicalType::BOOL());
+ column_types.emplace_back(LogicalType::INT64());
+ column_types.emplace_back(LogicalType::LIST(LogicalType::STRING()));
+ column_types.emplace_back(LogicalType::INT64());
+ column_types.emplace_back(LogicalType::INT64());
+ column_types.emplace_back(LogicalType::INT64());
+ column_types.emplace_back(LogicalType::STRING());
+ column_types.emplace_back(LogicalType::STRING());
+ break;
+ }
+ default:
+ throw BinderException("Unsupported GraphAr metadata type.");
+ }
+
+ column_names = TableFunction::extractYieldVariables(column_names, input->yieldVariables);
+ auto columns = input->binder->createVariables(column_names, column_types);
+ return std::make_unique<GrapharMetadataBindData>(std::move(columns), context,
+ std::move(graph_info), metadata_type, table_name, std::move(column_types),
+ std::move(column_names), std::move(vertex_info), std::move(edge_info));
+}
+
+std::unique_ptr<TableFuncSharedState> initGrapharMetadataSharedState(
+ [[maybe_unused]] const TableFuncInitSharedStateInput& input) {
+ return std::make_unique<GrapharMetadataSharedState>(false);
+}
+
+offset_t metadataTableFunc(const TableFuncInput& input, TableFuncOutput& output) {
+ auto bindData = input.bindData->constPtrCast<GrapharMetadataBindData>();
+ auto sharedState = input.sharedState->ptrCast<GrapharMetadataSharedState>();
+ auto& table_name = bindData->table_name;
+ auto& graph_info = bindData->graph_info;
+ auto& vertex_info = bindData->vertex_info;
+ auto& edge_info = bindData->edge_info;
+ auto metadata_type = bindData->metadata_type;
+
+ std::lock_guard<std::mutex> lock{sharedState->mtx};
+ if (sharedState->finished) {
+ output.dataChunk.state->getSelVectorUnsafe().setSelSize(0);
+ return output.dataChunk.state->getSelVector().getSelSize();
+ }
+ sharedState->finished = true;
+
+ // Helper: Flatten property groups into a vector of "name|type|is_primary" strings.
+ // The return value is a string vector that the caller writes into the data vector of the ListVector in order.
+ auto collectPropertyStrings = [](const std::vector<std::shared_ptr<graphar::PropertyGroup>>& property_groups) {
+ std::vector<std::string> out;
+ // Calculate the total number of properties and pre-allocate.
+ size_t total = 0;
+ for (const auto& group : property_groups) {
+ const auto& properties = group->GetProperties();
+ total += properties.size();
+ }
+ out.reserve(total);
+ for (const auto& group : property_groups) {
+ const auto& properties = group->GetProperties();
+ for (const auto& property : properties) {
+ out.emplace_back(property.name + METADATA_SEPARATOR +
+ property.type->ToTypeName() + METADATA_SEPARATOR +
+ (property.is_primary ? "true" : "false"));
+ }
+ }
+ return out;
+ };
+
+ // we come here only once.
+ switch (metadata_type) {
+ case GrapharMetadataType::GRAPH: {
+ if (graph_info == nullptr) {
+ throw BinderException("GraphAr's GraphInfo is null.");
+ }
+
+ // graph_name
+ auto& graph_name_vec = output.dataChunk.getValueVectorMutable(0);
+ graph_name_vec.setValue(0, graph_info->GetName());
+
+ // vertex_type_num
+ auto& vertex_type_num_vec = output.dataChunk.getValueVectorMutable(1);
+ size_t vertex_type_num = graph_info->GetVertexInfos().size();
+ vertex_type_num_vec.setValue(0, static_cast<int32_t>(vertex_type_num));
+
+ // edge_types_num
+ auto& edge_type_num_vec = output.dataChunk.getValueVectorMutable(2);
+ size_t edge_type_num = graph_info->GetEdgeInfos().size();
+ edge_type_num_vec.setValue(0, static_cast<int32_t>(edge_type_num));
+
+ // vertex_types
+
+ // Get the List value vector for the "vertex_types" column.
+ // A ListVector stores, for each row, a list_entry_t {offset, length}
+ // that points into a shared "data vector" containing the flattened elements.
+ auto& vertex_types_vec = output.dataChunk.getValueVectorMutable(3);
+
+ // Clear any previous auxiliary state (offsets/lengths, etc.) so we can
+ // populate the list from scratch for this execution.
+ vertex_types_vec.resetAuxiliaryBuffer();
+
+ // Obtain the underlying data vector that holds all list elements
+ // (for string lists this will be a StringVector). We must also reset
+ // its auxiliary buffer to start writing fresh element entries.
+ auto vertex_types_data_vec = ListVector::getDataVector(&vertex_types_vec);
+ vertex_types_data_vec->resetAuxiliaryBuffer();
+
+ // Retrieve the vertex info entries from graph_info and determine
+ // how many vertex type strings we need to store.
+ auto vertexInfos = graph_info->GetVertexInfos();
+ size_t num_vertex_types = vertexInfos.size();
+
+ // Reserve a contiguous slot in the data vector for this row's list
+ // elements. addList returns a list_entry_t containing the starting
+ // offset in the data vector and the length (num_vertex_types).
+ // If num_vertex_types == 0, the returned entry will have length 0.
+ auto resultList = ListVector::addList(&vertex_types_vec, num_vertex_types);
+
+ // Associate the list_entry_t with the output row 0 so that the row
+ // will reference the allocated slice in the shared data vector.
+ vertex_types_vec.setValue<list_entry_t>(0, resultList);
+
+ // Fill the reserved slice in the data vector with the per-vertex-type strings.
+ // We write each string at index (resultList.offset + i).
+ // Note: vertexInfos elements are accessed via pointer (->); adjust if they are objects.
+ for (size_t i = 0; i < num_vertex_types; ++i) {
+ // Convert the vertex type to std::string (or appropriate string view)
+ // and append it into the shared data vector at the correct position.
+ StringVector::addString(vertex_types_data_vec, resultList.offset + i,
+ vertexInfos[i]->GetType());
+ }
+
+ // End of vertex_types population. The ListVector now has, for row 0,
+ // an entry pointing to a contiguous range in vertex_types_data_vec
+ // containing the vertex type strings.
+
+ // edge_types
+ auto& edge_types_vec = output.dataChunk.getValueVectorMutable(4);
+ edge_types_vec.resetAuxiliaryBuffer();
+ auto edge_types_data_vec = ListVector::getDataVector(&edge_types_vec);
+ edge_types_data_vec->resetAuxiliaryBuffer();
+ auto edgeInfos = graph_info->GetEdgeInfos();
+ size_t num_edge_types = edgeInfos.size();
+ auto edgeResultList = ListVector::addList(&edge_types_vec, num_edge_types);
+ edge_types_vec.setValue<list_entry_t>(0, edgeResultList);
+ for (size_t i = 0; i < num_edge_types; ++i) {
+ StringVector::addString(edge_types_data_vec, edgeResultList.offset + i,
+ edgeInfos[i]->GetSrcType() + REGULAR_SEPARATOR + edgeInfos[i]->GetEdgeType() +
+ REGULAR_SEPARATOR + edgeInfos[i]->GetDstType());
+ }
+
+ // prefix
+ auto& prefix_vec = output.dataChunk.getValueVectorMutable(5);
+ prefix_vec.setValue(0, graph_info->GetPrefix());
+
+ // version
+ auto& version_vec = output.dataChunk.getValueVectorMutable(6);
+ version_vec.setValue(0, graph_info->version()->ToString());
+
+ break;
+ }
+ case GrapharMetadataType::VERTEX: {
+ if (graph_info == nullptr || vertex_info == nullptr) {
+ throw BinderException("GraphAr's GraphInfo or VertexInfo is null.");
+ }
+
+ auto vertex_collection = VerticesCollection::Make(graph_info, table_name).value();
+
+ // vertex_type
+ auto& vertex_type_vec = output.dataChunk.getValueVectorMutable(0);
+ vertex_type_vec.setValue(0, bindData->table_name);
+
+ // vertex_count
+ auto& vertex_count_vec = output.dataChunk.getValueVectorMutable(1);
+ vertex_count_vec.setValue(0, static_cast<int64_t>(vertex_collection->size()));
+
+ // property_names|types|is_primary
+ auto& property_names_vec = output.dataChunk.getValueVectorMutable(2);
+ property_names_vec.resetAuxiliaryBuffer();
+ auto property_names_data_vec = ListVector::getDataVector(&property_names_vec);
+ property_names_data_vec->resetAuxiliaryBuffer();
+ const auto property_groups = vertex_info->GetPropertyGroups();
+ auto property_strings = collectPropertyStrings(property_groups);
+ size_t num_properties = property_strings.size();
+
+ auto propertyResultList = ListVector::addList(&property_names_vec, num_properties);
+ property_names_vec.setValue<list_entry_t>(0, propertyResultList);
+
+ // Write data vector in sequence at once
+ for (size_t i = 0; i < num_properties; ++i) {
+ StringVector::addString(property_names_data_vec, propertyResultList.offset + i,
+ property_strings[i]);
+ }
+
+ // chunk_size
+ auto& chunk_size_vec = output.dataChunk.getValueVectorMutable(3);
+ chunk_size_vec.setValue(0, static_cast<int64_t>(vertex_info->GetChunkSize()));
+
+ // prefix
+ auto& prefix_vec = output.dataChunk.getValueVectorMutable(4);
+ prefix_vec.setValue(0, graph_info->GetPrefix());
+
+ // version
+ auto& version_vec = output.dataChunk.getValueVectorMutable(5);
+ version_vec.setValue(0, graph_info->version()->ToString());
+
+ break;
+ }
+ case GrapharMetadataType::EDGE: {
+ if (graph_info == nullptr || edge_info == nullptr) {
+ throw BinderException("GraphAr's GraphInfo or EdgeInfo is null.");
+ }
+
+ auto edge_collection = EdgesCollection::Make(graph_info, edge_info->GetSrcType(),
+ edge_info->GetEdgeType(), edge_info->GetDstType(), AdjListType::ordered_by_source)
+ .value();
+
+ // edge_type
+ auto& edge_type_vec = output.dataChunk.getValueVectorMutable(0);
+ edge_type_vec.setValue(0, bindData->table_name);
+
+ // src_type
+ auto& src_type_vec = output.dataChunk.getValueVectorMutable(1);
+ src_type_vec.setValue(0, edge_info->GetSrcType());
+
+ // dst_type
+ auto& dst_type_vec = output.dataChunk.getValueVectorMutable(2);
+ dst_type_vec.setValue(0, edge_info->GetDstType());
+
+ // directed
+ auto& directed_vec = output.dataChunk.getValueVectorMutable(3);
+ directed_vec.setValue(0, edge_info->IsDirected());
+
+ // edge_count
+ auto& edge_count_vec = output.dataChunk.getValueVectorMutable(4);
+ edge_count_vec.setValue(0, static_cast<int64_t>(edge_collection->size()));
+
+ // property_names|types|is_primary
+ auto& property_names_vec = output.dataChunk.getValueVectorMutable(5);
+ property_names_vec.resetAuxiliaryBuffer();
+ auto property_names_data_vec = ListVector::getDataVector(&property_names_vec);
+ property_names_data_vec->resetAuxiliaryBuffer();
+ const auto property_groups = edge_info->GetPropertyGroups();
+ auto property_strings = collectPropertyStrings(property_groups);
+ size_t num_properties = property_strings.size();
+
+ auto propertyResultList = ListVector::addList(&property_names_vec, num_properties);
+ property_names_vec.setValue<list_entry_t>(0, propertyResultList);
+
+ for (size_t i = 0; i < num_properties; ++i) {
+ StringVector::addString(property_names_data_vec, propertyResultList.offset + i,
+ property_strings[i]);
+ }
+
+ // chunk_size
+ auto& chunk_size_vec = output.dataChunk.getValueVectorMutable(6);
+ chunk_size_vec.setValue(0, static_cast<int64_t>(edge_info->GetChunkSize()));
+
+ // src_chunk_size
+ auto& src_chunk_size_vec = output.dataChunk.getValueVectorMutable(7);
+ src_chunk_size_vec.setValue(0, static_cast<int64_t>(edge_info->GetSrcChunkSize()));
+
+ // dst_chunk_size
+ auto& dst_chunk_size_vec = output.dataChunk.getValueVectorMutable(8);
+ dst_chunk_size_vec.setValue(0, static_cast<int64_t>(edge_info->GetDstChunkSize()));
+
+ // prefix
+ auto& prefix_vec = output.dataChunk.getValueVectorMutable(9);
+ prefix_vec.setValue(0, graph_info->GetPrefix());
+
+ // version
+ auto& version_vec = output.dataChunk.getValueVectorMutable(10);
+ version_vec.setValue(0, graph_info->version()->ToString());
+
+ break;
+ }
+ default:
+ throw BinderException("Unsupported GraphAr metadata type.");
+ }
+
+ output.dataChunk.state->getSelVectorUnsafe().setSelSize(1);
+ return output.dataChunk.state->getSelVector().getSelSize();
+}
+
+function_set GrapharMetadataFunction::getFunctionSet() {
+ function_set functionSet;
+ auto function = std::make_unique<TableFunction>(name, std::vector{LogicalTypeID::STRING});
+ function->tableFunc = metadataTableFunc;
+ function->bindFunc = metadataBindFunc;
+ function->initSharedStateFunc = initGrapharMetadataSharedState;
+ function->initLocalStateFunc = function::TableFunction::initEmptyLocalState;
+ functionSet.push_back(std::move(function));
+ return functionSet;
+}
+
+} // namespace graphar_extension
+} // namespace kuzu
diff --git a/cpp/extensions/kuzu-extension/src/function/graphar_scan.cpp b/cpp/extensions/kuzu-extension/src/function/graphar_scan.cpp
new file mode 100644
index 00000000..9915f5ed
--- /dev/null
+++ b/cpp/extensions/kuzu-extension/src/function/graphar_scan.cpp
@@ -0,0 +1,181 @@
+#include "function/graphar_scan.h"
+
+namespace kuzu {
+namespace graphar_extension {
+
+using namespace function;
+using namespace common;
+
+GrapharScanSharedState::GrapharScanSharedState(
+ graphar::Result<std::shared_ptr<graphar::VerticesCollection>> maybeVerticesCollection,
+ graphar::Result<std::shared_ptr<graphar::EdgesCollection>> maybeEdgesCollection,
+ graphar::Result<std::shared_ptr<graphar::VerticesCollection>> maybeFromVerticesCollection,
+ graphar::Result<std::shared_ptr<graphar::VerticesCollection>> maybeToVerticesCollection,
+ uint64_t max_threads, bool is_edge)
+ : maybe_vertices_collection{std::move(maybeVerticesCollection)},
+ maybe_edges_collection{std::move(maybeEdgesCollection)},
+ maybe_from_vertices_collection{std::move(maybeFromVerticesCollection)},
+ maybe_to_vertices_collection{std::move(maybeToVerticesCollection)} {
+ if (!is_edge) {
+ collection_count = maybe_vertices_collection.value()->size();
+ } else {
+ collection_count = maybe_edges_collection.value()->size();
+ }
+ KU_ASSERT(collection_count != 0);
+ batch_size = max_threads % collection_count == 0 ?
+ std::max<size_t>(1, collection_count / max_threads) :
+ std::max<size_t>(1, collection_count / max_threads + 1);
+ batch_size = batch_size > DEFAULT_VECTOR_CAPACITY ? DEFAULT_VECTOR_CAPACITY : batch_size;
+ next_index.store(0);
+
+ // If it is edge, pre-compute the starting point of each batch's EdgeIter in the init stage.
+ if (is_edge && maybe_edges_collection.has_value()) {
+ auto edges = maybe_edges_collection.value();
+ size_t num_batches = (collection_count + batch_size - 1) / batch_size;
+ edge_batch_iters.reserve(num_batches);
+
+ // Single sequential scan: Walk from begin() to end(), and push_back a copy at the first
+ // index of each batch.
+ auto iter = edges->begin();
+ for (size_t idx = 0; idx < collection_count; ++idx) {
+ if (idx % batch_size == 0) {
+ edge_batch_iters.emplace_back(iter); // copy constructor
+ }
+ ++iter;
+ }
+ }
+}
+
+std::unique_ptr<TableFuncSharedState> initGrapharScanSharedState(
+ const TableFuncInitSharedStateInput& input) {
+ auto grapharScanBindData = input.bindData->constPtrCast<GrapharScanBindData>();
+ graphar::Result<std::shared_ptr<graphar::VerticesCollection>> maybe_vertices_collection = {};
+ graphar::Result<std::shared_ptr<graphar::EdgesCollection>> maybe_edges_collection = {};
+ graphar::Result<std::shared_ptr<graphar::VerticesCollection>> maybe_from_vertices_collection =
+ {};
+ graphar::Result<std::shared_ptr<graphar::VerticesCollection>> maybe_to_vertices_collection = {};
+
+ if (!grapharScanBindData->is_edge) {
+ maybe_vertices_collection = graphar::VerticesCollection::Make(
+ grapharScanBindData->graph_info, grapharScanBindData->table_name);
+ } else {
+ // parse table_name into src.edge.dst
+ std::string src, edge, dst;
+ if (!tryParseEdgeTableName(grapharScanBindData->table_name, src, edge, dst)) {
+ throw BinderException("Edge table name " + grapharScanBindData->table_name +
+ " is invalid. It should be in the format of "
+ "<source>_<edge>_<destination>.");
+ }
+
+ maybe_edges_collection = graphar::EdgesCollection::Make(grapharScanBindData->graph_info,
+ src, edge, dst, graphar::AdjListType::ordered_by_source);
+ maybe_from_vertices_collection =
+ graphar::VerticesCollection::Make(grapharScanBindData->graph_info, src);
+ maybe_to_vertices_collection =
+ graphar::VerticesCollection::Make(grapharScanBindData->graph_info, dst);
+ }
+ return std::make_unique<graphar_extension::GrapharScanSharedState>(
+ std::move(maybe_vertices_collection), std::move(maybe_edges_collection),
+ std::move(maybe_from_vertices_collection), std::move(maybe_to_vertices_collection),
+ grapharScanBindData->max_threads, grapharScanBindData->is_edge);
+}
+
+offset_t tableFunc(const TableFuncInput& input, TableFuncOutput& output) {
+ auto grapharSharedState =
+ input.sharedState->ptrCast<graphar_extension::GrapharScanSharedState>();
+ auto grapharScanBindData =
+ input.bindData->constPtrCast<graphar_extension::GrapharScanBindData>();
+
+ if (!grapharScanBindData->is_edge) {
+ auto& column_setters = grapharScanBindData->vertex_column_setters;
+ auto vertices = grapharSharedState->maybe_vertices_collection.value();
+ size_t vertices_count = grapharSharedState->collection_count;
+ size_t batch_size = grapharSharedState->batch_size;
+
+ // Reserve a batch of indices atomically
+ size_t start =
+ grapharSharedState->next_index.fetch_add(batch_size, std::memory_order_relaxed);
+ if (start >= vertices_count) {
+ // no more rows
+ output.dataChunk.state->getSelVectorUnsafe().setSelSize(0);
+ return 0;
+ }
+ size_t end = std::min(start + batch_size, vertices_count);
+
+ idx_t count = 0;
+ // create a local iterator for this index (each worker/thread has its own local iterator)
+ auto it = vertices->begin() + start;
+ for (size_t idx = start; idx < end; ++idx) {
+ // Complete all column writes using the setter generated in the bind stage (without
+ // switch)
+ for (size_t ci = 0; ci < column_setters.size(); ++ci) {
+ column_setters[ci](it, output, count);
+ }
+ ++it;
+ count++;
+ }
+
+ output.dataChunk.state->getSelVectorUnsafe().setSelSize(count);
+ return output.dataChunk.state->getSelVector().getSelSize();
+ } else {
+ auto& column_names = grapharScanBindData->column_names;
+ auto& column_setters = grapharScanBindData->edge_column_setters;
+ auto edges = grapharSharedState->maybe_edges_collection.value();
+ size_t edges_count = grapharSharedState->collection_count;
+ size_t batch_size = grapharSharedState->batch_size;
+
+ // Reserve a batch of indices atomically
+ size_t start =
+ grapharSharedState->next_index.fetch_add(batch_size, std::memory_order_relaxed);
+ if (start >= edges_count) {
+ // no more rows
+ output.dataChunk.state->getSelVectorUnsafe().setSelSize(0);
+ return 0;
+ }
+ size_t end = std::min(start + batch_size, edges_count);
+
+ // Use the pre-computed batch start iter in SharedState to avoid advancing from the
+ // beginning each time.
+ size_t batch_id = start / batch_size;
+ KU_ASSERT(batch_id < grapharSharedState->edge_batch_iters.size());
+
+ idx_t count = 0;
+ auto from_vertices = grapharSharedState->maybe_from_vertices_collection.value();
+ auto to_vertices = grapharSharedState->maybe_to_vertices_collection.value();
+
+ auto it = grapharSharedState->edge_batch_iters[batch_id];
+ for (size_t idx = start; idx < end; ++idx) {
+ for (size_t ci = 0; ci < column_setters.size(); ++ci) {
+ if (StringUtils::caseInsensitiveEquals(column_names[ci], FROM_COL_NAME) ||
+ StringUtils::caseInsensitiveEquals(column_names[ci], INTERNAL_FROM_COL_NAME)) {
+ column_setters[ci](it, output, count, from_vertices); // from setter
+ } else if (StringUtils::caseInsensitiveEquals(column_names[ci], TO_COL_NAME) ||
+ StringUtils::caseInsensitiveEquals(column_names[ci],
+ INTERNAL_TO_COL_NAME)) {
+ column_setters[ci](it, output, count, to_vertices); // to setter
+ } else {
+ column_setters[ci](it, output, count, nullptr); // other setter
+ }
+ }
+ ++it;
+ count++;
+ }
+
+ output.dataChunk.state->getSelVectorUnsafe().setSelSize(count);
+ return output.dataChunk.state->getSelVector().getSelSize();
+ }
+}
+
+function_set GrapharScanFunction::getFunctionSet() {
+ function_set functionSet;
+ auto function = std::make_unique<TableFunction>(name, std::vector{LogicalTypeID::STRING});
+ function->tableFunc = tableFunc;
+ function->bindFunc = bindFunc;
+ function->initSharedStateFunc = initGrapharScanSharedState;
+ function->initLocalStateFunc = TableFunction::initEmptyLocalState;
+ functionSet.push_back(std::move(function));
+ return functionSet;
+}
+
+} // namespace graphar_extension
+} // namespace kuzu
diff --git a/cpp/extensions/kuzu-extension/src/function/graphar_scan_bindfunc.cpp b/cpp/extensions/kuzu-extension/src/function/graphar_scan_bindfunc.cpp
new file mode 100644
index 00000000..4677b4e0
--- /dev/null
+++ b/cpp/extensions/kuzu-extension/src/function/graphar_scan_bindfunc.cpp
@@ -0,0 +1,518 @@
+#include "function/graphar_scan.h"
+
+namespace kuzu {
+namespace graphar_extension {
+
+using namespace function;
+using namespace common;
+
+// Vertex setter maker for properties
+template<typename T>
+VertexColumnSetter makeTypedVertexSetter(uint64_t fieldIdx, std::string colName) {
+ return [fieldIdx, colName = std::move(colName)](graphar::VertexIter& it,
+ function::TableFuncOutput& output, kuzu::common::idx_t row) {
+ auto res = it.property<T>(colName);
+ auto& vec = output.dataChunk.getValueVectorMutable(fieldIdx);
+ vec.setValue(row, res.value());
+ };
+}
+
+// Vertex setter maker for internal_id
+VertexColumnSetter makeInternalIdVertexSetter(uint64_t fieldIdx, std::string colName) {
+ return [fieldIdx, colName = std::move(colName)](graphar::VertexIter& it,
+ function::TableFuncOutput& output, kuzu::common::idx_t row) {
+ auto id = it.id();
+ auto& vec = output.dataChunk.getValueVectorMutable(fieldIdx);
+ vec.setValue(row, static_cast<int64_t>(id));
+ };
+}
+
+template<>
+VertexColumnSetter makeTypedVertexSetter<list_entry_t>([[maybe_unused]] uint64_t fieldIdx,
+ [[maybe_unused]] std::string colName) {
+ throw NotImplementedException("List type is not supported in graphar scan.");
+}
+
+// Edge setter maker for properties
+template<typename T>
+EdgeColumnSetter makeTypedEdgeSetter(uint64_t fieldIdx, std::string colName) {
+ return [fieldIdx, colName = std::move(colName)](graphar::EdgeIter& it,
+ function::TableFuncOutput& output, kuzu::common::idx_t row,
+ [[maybe_unused]] std::shared_ptr<graphar::VerticesCollection> unused) {
+ auto res = it.property<T>(colName);
+ auto& vec = output.dataChunk.getValueVectorMutable(fieldIdx);
+ vec.setValue(row, res.value());
+ };
+}
+
+template<>
+EdgeColumnSetter makeTypedEdgeSetter<list_entry_t>([[maybe_unused]] uint64_t fieldIdx,
+ [[maybe_unused]] std::string colName) {
+ throw NotImplementedException("List type is not supported in graphar scan.");
+}
+
+// Edge setter for "from" (source) and "to" (destination)
+template<typename T>
+EdgeColumnSetter makeFromSetter(uint64_t fieldIdx, std::string colName) {
+ return [fieldIdx, colName = std::move(colName)](graphar::EdgeIter& it,
+ function::TableFuncOutput& output, kuzu::common::idx_t row,
+ std::shared_ptr<graphar::VerticesCollection> from_vertices) {
+ graphar::IdType src = it.source();
+ auto vertex_it = from_vertices->find(src);
+ auto res = vertex_it.property<T>(colName);
+ auto& vec = output.dataChunk.getValueVectorMutable(fieldIdx);
+ vec.setValue(row, res.value());
+ };
+}
+
+template<typename T>
+EdgeColumnSetter makeToSetter(uint64_t fieldIdx, std::string colName) {
+ return [fieldIdx, colName = std::move(colName)](graphar::EdgeIter& it,
+ function::TableFuncOutput& output, kuzu::common::idx_t row,
+ std::shared_ptr<graphar::VerticesCollection> to_vertices) {
+ graphar::IdType dst = it.destination();
+ auto vertex_it = to_vertices->find(dst);
+ auto res = vertex_it.property<T>(colName);
+ auto& vec = output.dataChunk.getValueVectorMutable(fieldIdx);
+ vec.setValue(row, res.value());
+ };
+}
+
+// Edge setter for "internal_from" (source) and "internal_to" (destination)
+EdgeColumnSetter makeInternalFromSetter(uint64_t fieldIdx) {
+ return
+ [fieldIdx](graphar::EdgeIter& it, function::TableFuncOutput& output,
+ kuzu::common::idx_t row, std::shared_ptr<graphar::VerticesCollection> from_vertices) {
+ graphar::IdType src = it.source();
+ auto vertex_it = from_vertices->find(src);
+ auto res = vertex_it.id();
+ auto& vec = output.dataChunk.getValueVectorMutable(fieldIdx);
+ vec.setValue(row, static_cast<int64_t>(res));
+ };
+}
+
+EdgeColumnSetter makeInternalToSetter(uint64_t fieldIdx) {
+ return [fieldIdx](graphar::EdgeIter& it, function::TableFuncOutput& output,
+ kuzu::common::idx_t row, std::shared_ptr<graphar::VerticesCollection> to_vertices) {
+ graphar::IdType dst = it.destination();
+ auto vertex_it = to_vertices->find(dst);
+ auto res = vertex_it.id();
+ auto& vec = output.dataChunk.getValueVectorMutable(fieldIdx);
+ vec.setValue(row, static_cast<int64_t>(res));
+ };
+}
+
+static const std::unordered_map<LogicalTypeID,
+ std::function<VertexColumnSetter(uint64_t, std::string)>>
+ vertexSetterFactory = {
+ {LogicalTypeID::INT64,
+ [](uint64_t idx, std::string col) {
+ return makeTypedVertexSetter<int64_t>(idx, std::move(col));
+ }},
+ {LogicalTypeID::INT32,
+ [](uint64_t idx, std::string col) {
+ return makeTypedVertexSetter<int32_t>(idx, std::move(col));
+ }},
+ {LogicalTypeID::DOUBLE,
+ [](uint64_t idx, std::string col) {
+ return makeTypedVertexSetter<double>(idx, std::move(col));
+ }},
+ {LogicalTypeID::FLOAT,
+ [](uint64_t idx, std::string col) {
+ return makeTypedVertexSetter<float>(idx, std::move(col));
+ }},
+ {LogicalTypeID::STRING,
+ [](uint64_t idx, std::string col) {
+ return makeTypedVertexSetter<std::string>(idx, std::move(col));
+ }},
+ {LogicalTypeID::BOOL,
+ [](uint64_t idx, std::string col) {
+ return makeTypedVertexSetter<bool>(idx, std::move(col));
+ }},
+ {LogicalTypeID::DATE,
+ [](uint64_t idx, std::string col) {
+ return makeTypedVertexSetter<date_t>(idx, std::move(col));
+ }},
+ {LogicalTypeID::TIMESTAMP,
+ [](uint64_t idx, std::string col) {
+ return makeTypedVertexSetter<timestamp_t>(idx, std::move(col));
+ }},
+ {LogicalTypeID::LIST,
+ [](uint64_t idx, std::string col) {
+ return makeTypedVertexSetter<list_entry_t>(idx, std::move(col));
+ }},
+};
+
+static const std::unordered_map<LogicalTypeID,
+ std::function<EdgeColumnSetter(uint64_t, std::string)>>
+ edgeSetterFactory = {
+ {LogicalTypeID::INT64,
+ [](uint64_t idx, std::string col) {
+ return makeTypedEdgeSetter<int64_t>(idx, std::move(col));
+ }},
+ {LogicalTypeID::INT32,
+ [](uint64_t idx, std::string col) {
+ return makeTypedEdgeSetter<int32_t>(idx, std::move(col));
+ }},
+ {LogicalTypeID::DOUBLE,
+ [](uint64_t idx, std::string col) {
+ return makeTypedEdgeSetter<double>(idx, std::move(col));
+ }},
+ {LogicalTypeID::FLOAT,
+ [](uint64_t idx, std::string col) {
+ return makeTypedEdgeSetter<float>(idx, std::move(col));
+ }},
+ {LogicalTypeID::STRING,
+ [](uint64_t idx, std::string col) {
+ return makeTypedEdgeSetter<std::string>(idx, std::move(col));
+ }},
+ {LogicalTypeID::BOOL,
+ [](uint64_t idx, std::string col) {
+ return makeTypedEdgeSetter<bool>(idx, std::move(col));
+ }},
+ {LogicalTypeID::DATE,
+ [](uint64_t idx, std::string col) {
+ return makeTypedEdgeSetter<date_t>(idx, std::move(col));
+ }},
+ {LogicalTypeID::TIMESTAMP,
+ [](uint64_t idx, std::string col) {
+ return makeTypedEdgeSetter<timestamp_t>(idx, std::move(col));
+ }},
+ {LogicalTypeID::LIST,
+ [](uint64_t idx, std::string col) {
+ return makeTypedEdgeSetter<list_entry_t>(idx, std::move(col));
+ }},
+};
+
+static const std::unordered_map<LogicalTypeID,
+ std::function<EdgeColumnSetter(uint64_t, std::string)>>
+ fromSetterFactory = {
+ {LogicalTypeID::INT64,
+ [](uint64_t idx, std::string col) {
+ return makeFromSetter<int64_t>(idx, std::move(col));
+ }},
+ {LogicalTypeID::INT32,
+ [](uint64_t idx, std::string col) {
+ return makeFromSetter<int32_t>(idx, std::move(col));
+ }},
+ {LogicalTypeID::DOUBLE,
+ [](uint64_t idx, std::string col) {
+ return makeFromSetter<double>(idx, std::move(col));
+ }},
+ {LogicalTypeID::FLOAT,
+ [](uint64_t idx, std::string col) {
+ return makeFromSetter<float>(idx, std::move(col));
+ }},
+ {LogicalTypeID::STRING,
+ [](uint64_t idx, std::string col) {
+ return makeFromSetter<std::string>(idx, std::move(col));
+ }},
+};
+
+static const std::unordered_map<LogicalTypeID,
+ std::function<EdgeColumnSetter(uint64_t, std::string)>>
+ toSetterFactory = {
+ {LogicalTypeID::INT64,
+ [](uint64_t idx, std::string col) {
+ return makeToSetter<int64_t>(idx, std::move(col));
+ }},
+ {LogicalTypeID::INT32,
+ [](uint64_t idx, std::string col) {
+ return makeToSetter<int32_t>(idx, std::move(col));
+ }},
+ {LogicalTypeID::DOUBLE,
+ [](uint64_t idx, std::string col) {
+ return makeToSetter<double>(idx, std::move(col));
+ }},
+ {LogicalTypeID::FLOAT,
+ [](uint64_t idx, std::string col) { return makeToSetter<float>(idx, std::move(col)); }},
+ {LogicalTypeID::STRING,
+ [](uint64_t idx, std::string col) {
+ return makeToSetter<std::string>(idx, std::move(col));
+ }},
+};
+
+static void autoDetectVertexSchema([[maybe_unused]] main::ClientContext* context,
+ std::shared_ptr<graphar::GraphInfo> graph_info, std::string table_name,
+ std::vector<LogicalType>& types, std::vector<std::string>& names) {
+ auto vertexInfo = graph_info->GetVertexInfo(table_name);
+ if (!vertexInfo) {
+ throw BinderException("GraphAr's VertexInfo " + table_name + " does not exist as vertex.");
+ }
+
+ names.push_back("internal_id");
+ types.push_back(LogicalType::INT64());
+
+ // Construct the types and names from the vertex info.
+ for (auto& property_group : vertexInfo->GetPropertyGroups()) {
+ for (const auto& property : property_group->GetProperties()) {
+ names.push_back(property.name);
+ types.push_back(grapharTypeToKuzuType(property.type));
+ }
+ }
+}
+
+static void autoDetectEdgeSchema([[maybe_unused]] main::ClientContext* context,
+ std::shared_ptr<graphar::GraphInfo> graph_info, const std::string& table_name,
+ std::vector<LogicalType>& types, std::vector<std::string>& names,
+ std::unordered_map<std::string, std::string>& edges_from_to_mapping) {
+ std::string src_type, edge_type, dst_type;
+ if (!tryParseEdgeTableName(table_name, src_type, edge_type, dst_type)) {
+ throw BinderException("Edge table_name must be specified in `src_edge_dst` format "
+ "(supported separators: '_'). Given: " +
+ table_name);
+ }
+
+ // Use GraphInfo to get EdgeInfo
+ auto edgeInfo = graph_info->GetEdgeInfo(src_type, edge_type, dst_type);
+ if (!edgeInfo) {
+ throw BinderException("GraphAr's EdgeInfo does not exist for " + table_name +
+ " (parsed as " + src_type + REGULAR_SEPARATOR + edge_type +
+ REGULAR_SEPARATOR + dst_type + ").");
+ }
+
+ auto from_vertex_info = graph_info->GetVertexInfo(src_type);
+ auto to_vertex_info = graph_info->GetVertexInfo(dst_type);
+ if (!from_vertex_info) {
+ throw BinderException(
+ "GraphAr's VertexInfo does not exist for edge source type " + src_type + ".");
+ }
+ if (!to_vertex_info) {
+ throw BinderException(
+ "GraphAr's VertexInfo does not exist for edge destination type " + dst_type + ".");
+ }
+
+ // Prepend from/to columns
+ names.push_back(FROM_COL_NAME);
+ // Find from column mapping name in the from_vertex_info's primary keys
+ for (const auto& propertyGroup : from_vertex_info->GetPropertyGroups()) {
+ for (const auto& property : propertyGroup->GetProperties()) {
+ if (property.is_primary) {
+ edges_from_to_mapping.insert({FROM_COL_NAME, property.name});
+ break;
+ }
+ }
+ }
+ types.push_back(LogicalType::INT64());
+
+ names.push_back(TO_COL_NAME);
+ // Find to column mapping name in the to_vertex_info's primary keys
+ for (const auto& propertyGroup : to_vertex_info->GetPropertyGroups()) {
+ for (const auto& property : propertyGroup->GetProperties()) {
+ if (property.is_primary) {
+ edges_from_to_mapping.insert({TO_COL_NAME, property.name});
+ break;
+ }
+ }
+ }
+ types.push_back(LogicalType::INT64());
+
+ // Add internal_from/internal_to edge columns (optional)
+ // internal_from/to is optional, because they are stored in internal_id of vertices.
+ // names.push_back(INTERNAL_FROM_COL_NAME);
+ // types.push_back(LogicalType::INT64());
+ // names.push_back(INTERNAL_TO_COL_NAME);
+ // types.push_back(LogicalType::INT64());
+
+ // Add the edge properties except from/to
+ for (auto& property_group : edgeInfo->GetPropertyGroups()) {
+ for (const auto& property : property_group->GetProperties()) {
+ names.push_back(property.name);
+ types.push_back(grapharTypeToKuzuType(property.type));
+ }
+ }
+}
+
+GrapharScanBindData::GrapharScanBindData(binder::expression_vector columns,
+ common::FileScanInfo fileScanInfo, main::ClientContext* context,
+ std::shared_ptr<graphar::GraphInfo> graph_info, std::string table_name,
+ std::vector<std::string> column_names, std::vector<kuzu::common::LogicalType> column_types,
+ bool is_edge)
+ : ScanFileBindData{std::move(columns), 0 /* numRows */, std::move(fileScanInfo), context},
+ graph_info{std::move(graph_info)}, column_info{std::make_shared<KuzuColumnInfo>(
+ column_names)},
+ table_name{std::move(table_name)}, column_names{std::move(column_names)},
+ column_types{std::move(column_types)}, is_edge(is_edge) {
+ if (!is_edge) {
+ this->vertex_column_setters.reserve(this->column_names.size());
+ for (size_t i = 0; i < this->column_names.size(); ++i) {
+ if (StringUtils::caseInsensitiveEquals(this->column_names[i], "internal_id")) {
+ // special-case internal_id
+ this->vertex_column_setters.push_back(makeInternalIdVertexSetter(
+ getFieldIdx(this->column_names[i]), this->column_names[i]));
+ continue;
+ }
+ auto typeID = this->column_types[i].getLogicalTypeID();
+ uint64_t fieldIdx = getFieldIdx(this->column_names[i]);
+ KU_ASSERT(fieldIdx != UINT64_MAX);
+ auto it = vertexSetterFactory.find(typeID);
+ if (it == vertexSetterFactory.end()) {
+ throw NotImplementedException{
+ "Unsupported column type in GrapharScan bind (vertex): " +
+ std::to_string((int)typeID)};
+ }
+ this->vertex_column_setters.push_back(it->second(fieldIdx, this->column_names[i]));
+ }
+ } else {
+ this->edge_column_setters.reserve(this->column_names.size());
+ for (size_t i = 0; i < this->column_names.size(); ++i) {
+ uint64_t fieldIdx = getFieldIdx(this->column_names[i]);
+ KU_ASSERT(fieldIdx != UINT64_MAX);
+ // special-case from/to
+ if (StringUtils::caseInsensitiveEquals(this->column_names[i], FROM_COL_NAME)) {
+ LogicalTypeID typeID = this->column_types[i].getLogicalTypeID();
+ auto it = fromSetterFactory.find(typeID);
+ if (it == fromSetterFactory.end()) {
+ throw NotImplementedException{
+ "Unsupported column type in GrapharScan bind (from edge): " +
+ std::to_string((int)typeID)};
+ }
+ this->edge_column_setters.push_back(it->second(fieldIdx, "id"));
+ continue;
+ }
+
+ if (StringUtils::caseInsensitiveEquals(this->column_names[i], TO_COL_NAME)) {
+ LogicalTypeID typeID = this->column_types[i].getLogicalTypeID();
+ auto it = toSetterFactory.find(typeID);
+ if (it == toSetterFactory.end()) {
+ throw NotImplementedException{
+ "Unsupported column type in GrapharScan bind (to edge): " +
+ std::to_string((int)typeID)};
+ }
+ this->edge_column_setters.push_back(it->second(fieldIdx, "id"));
+ continue;
+ }
+
+ if (StringUtils::caseInsensitiveEquals(this->column_names[i], INTERNAL_FROM_COL_NAME)) {
+ // special-case internal_from
+ this->edge_column_setters.push_back(
+ makeInternalFromSetter(getFieldIdx(this->column_names[i])));
+ continue;
+ }
+
+ if (StringUtils::caseInsensitiveEquals(this->column_names[i], INTERNAL_TO_COL_NAME)) {
+ // special-case internal_to
+ this->edge_column_setters.push_back(
+ makeInternalToSetter(getFieldIdx(this->column_names[i])));
+ continue;
+ }
+
+ // regular edge property
+ LogicalTypeID typeID = this->column_types[i].getLogicalTypeID();
+ auto it = edgeSetterFactory.find(typeID);
+ if (it == edgeSetterFactory.end()) {
+ throw NotImplementedException{
+ "Unsupported column type in GrapharScan bind (edge): " +
+ std::to_string((int)typeID)};
+ }
+ this->edge_column_setters.push_back(it->second(fieldIdx, this->column_names[i]));
+ }
+ }
+ this->max_threads = context->getMaxNumThreadForExec();
+}
+
+KuzuColumnInfo::KuzuColumnInfo(std::vector<std::string> column_names) {
+ this->colNames = std::move(column_names);
+ idx_t colIdx = 0;
+ for (auto& colName : this->colNames) {
+ colNameToIdx.insert({colName, colIdx++});
+ }
+}
+
+uint64_t KuzuColumnInfo::getFieldIdx(std::string fieldName) const {
+ // For a small number of keys, probing a vector is faster than lookups in an unordered_map.
+ if (colNames.size() < 24) {
+ auto iter = std::find(colNames.begin(), colNames.end(), fieldName);
+ if (iter != colNames.end()) {
+ return iter - colNames.begin();
+ }
+ } else {
+ auto itr = colNameToIdx.find(fieldName);
+ if (itr != colNameToIdx.end()) {
+ return itr->second;
+ }
+ }
+ // From and to are case-insensitive for backward compatibility.
+ if (StringUtils::caseInsensitiveEquals(fieldName, FROM_COL_NAME)) {
+ return colNameToIdx.at(FROM_COL_NAME);
+ } else if (StringUtils::caseInsensitiveEquals(fieldName, TO_COL_NAME)) {
+ return colNameToIdx.at(TO_COL_NAME);
+ }
+ return UINT64_MAX;
+}
+
+std::unique_ptr<TableFuncBindData> bindFunc(main::ClientContext* context,
+ const TableFuncBindInput* input) {
+ auto scanInput = ku_dynamic_cast<ExtraScanTableFuncBindInput*>(input->extraInput.get());
+ std::string absolute_path = scanInput->fileScanInfo.getFilePath(0);
+ if (absolute_path.empty()) {
+ throw BinderException("GraphAr scan requires a valid file path.");
+ }
+
+ std::string table_name = scanInput->fileScanInfo.options.at("table_name").strVal;
+
+ // Load graph info from the file path
+ auto graph_info = graphar::GraphInfo::Load(absolute_path).value();
+ if (!graph_info) {
+ throw BinderException("GraphAr's GraphInfo could not be loaded from " + absolute_path);
+ }
+
+ std::vector<LogicalType> column_types;
+ std::vector<std::string> column_names;
+ std::unordered_map<std::string, std::string> edges_from_to_mapping;
+ bool is_edge = false;
+
+ auto vertex_infos = graph_info->GetVertexInfos();
+ auto edge_infos = graph_info->GetEdgeInfos();
+
+ // Try vertex first
+ bool found = false;
+ for (const auto& v_info : vertex_infos) {
+ if (v_info->GetType() == table_name) {
+ autoDetectVertexSchema(context, graph_info, table_name, column_types, column_names);
+ is_edge = false;
+ found = true;
+ break;
+ }
+ }
+
+ // Try edge if not vertex
+ if (!found) {
+ for (const auto& e_info : edge_infos) {
+ std::string src_type = e_info->GetSrcType();
+ std::string edge_type = e_info->GetEdgeType();
+ std::string dst_type = e_info->GetDstType();
+ std::string full_edge_name =
+ src_type + REGULAR_SEPARATOR + edge_type + REGULAR_SEPARATOR + dst_type;
+ if (full_edge_name == table_name) {
+ autoDetectEdgeSchema(context, graph_info, table_name, column_types, column_names,
+ edges_from_to_mapping);
+ is_edge = true;
+ found = true;
+ break;
+ }
+ }
+ }
+
+ KU_ASSERT(found);
+ KU_ASSERT(column_types.size() == column_names.size());
+
+ // ignore property type suffixes in column names.
+ // Examples:
+ // 'person-date' (vertex) -getFirstToken-> 'person'
+ // 'knows-timestamp' (edge) -getFirstToken-> 'knows'
+ std::vector<std::string> base_column_names;
+ for (auto& name : column_names) {
+ base_column_names.push_back(getFirstToken(name));
+ }
+
+ base_column_names =
+ TableFunction::extractYieldVariables(base_column_names, input->yieldVariables);
+ auto columns = input->binder->createVariables(base_column_names, column_types);
+ return std::make_unique<GrapharScanBindData>(std::move(columns), scanInput->fileScanInfo.copy(),
+ context, std::move(graph_info), std::move(table_name), std::move(column_names),
+ std::move(column_types), is_edge);
+}
+
+} // namespace graphar_extension
+} // namespace kuzu
diff --git a/cpp/extensions/kuzu-extension/src/include/function/graphar_export.h b/cpp/extensions/kuzu-extension/src/include/function/graphar_export.h
new file mode 100644
index 00000000..3f2735b5
--- /dev/null
+++ b/cpp/extensions/kuzu-extension/src/include/function/graphar_export.h
@@ -0,0 +1,349 @@
+#pragma once
+
+#include <optional>
+#include <variant>
+
+#include "common/exception/binder.h"
+#include "common/exception/not_implemented.h"
+#include "common/exception/runtime.h"
+#include "common/string_utils.h"
+#include "common/types/types.h"
+#include "common/vector/value_vector.h"
+#include "function/export/export_function.h"
+#include "function/function.h"
+#include "function/table/bind_data.h"
+#include "graphar/api/high_level_reader.h"
+#include "graphar/api/high_level_writer.h"
+#include "graphar/writer_util.h"
+#include "main/client_context.h"
+#include "utils/graphar_utils.h"
+
+namespace kuzu {
+namespace graphar_extension {
+
+using namespace kuzu::function;
+using namespace kuzu::common;
+using namespace graphar;
+
+// Scalar is a single atomic value type.
+using Scalar = std::variant<int64_t, int32_t, double, float, std::string, bool>;
+
+// Prop can be empty (monostate), a single scalar, or a list of scalars.
+using Prop = std::variant<std::monostate, Scalar, std::vector<Scalar>>;
+
+// schema metadata
+struct PropMeta {
+ std::string name;
+ Type type;
+ Cardinality card;
+};
+
+// row structure: store properties by schema index
+struct WriteRow {
+ std::vector<Prop> props; // length == schema.size()
+
+ WriteRow(size_t n) : props(n) {}
+};
+
+// helper: check whether a Scalar holds the exact Type (strict match)
+inline bool scalarMatchesType(const Scalar& scalar, Type type) {
+ switch (type) {
+ case Type::INT64:
+ case Type::TIMESTAMP:
+ return std::holds_alternative<int64_t>(scalar);
+ case Type::INT32:
+ case Type::DATE:
+ return std::holds_alternative<int32_t>(scalar);
+ case Type::DOUBLE:
+ return std::holds_alternative<double>(scalar);
+ case Type::FLOAT:
+ return std::holds_alternative<float>(scalar);
+ case Type::STRING:
+ return std::holds_alternative<std::string>(scalar);
+ case Type::BOOL:
+ return std::holds_alternative<bool>(scalar);
+ default:
+ throw common::NotImplementedException{"scalarMatchesType not implemented for type"};
+ }
+ return false;
+}
+
+// Convert a Scalar to a human-readable string for debugging.
+inline std::string scalarToString(const Scalar& s) {
+ return std::visit(
+ [](auto&& v) -> std::string {
+ using T = std::decay_t<decltype(v)>;
+ if constexpr (std::is_same_v<T, std::string>)
+ return v;
+ else if constexpr (std::is_same_v<T, bool>)
+ return v ? "true" : "false";
+ else {
+ std::ostringstream oss;
+ oss << v;
+ return oss.str();
+ }
+ },
+ s);
+}
+
+// Manage schema + row collection
+class WriteRowsBuffer {
+public:
+ // Construct with schema. Build name -> index map.
+ WriteRowsBuffer(const std::vector<PropMeta>& schema_) : schema(schema_) {
+ for (size_t i = 0; i < schema.size(); ++i)
+ name_to_idx[schema[i].name] = i;
+ }
+
+ // Create a new empty row and return its index.
+ size_t NewRow() {
+ rows.emplace_back(schema.size());
+ return rows.size() - 1;
+ }
+
+ // Set a single scalar property for a row. Enforce cardinality and type.
+ void SetProperty(size_t row_idx, const std::string& name, const Scalar& scalar) {
+ size_t idx = idxOf(name);
+ const auto& meta = schema[idx];
+ if (meta.card != Cardinality::SINGLE)
+ throw common::RuntimeException(
+ common::stringFormat("property '{}' is not SCALAR", name));
+ if (!scalarMatchesType(scalar, meta.type))
+ throw common::RuntimeException(
+ common::stringFormat("type mismatch for property '{}'", name));
+ rows.at(row_idx).props[idx] = scalar;
+ }
+
+ // Append a single element into a list-valued property.
+ void AddListElement(size_t row_idx, const std::string& name, const Scalar& scalar) {
+ size_t idx = idxOf(name);
+ const auto& meta = schema[idx];
+ if (meta.card != Cardinality::LIST)
+ throw common::RuntimeException(
+ common::stringFormat("Property '{}' is not LIST.", name));
+
+ if (!scalarMatchesType(scalar, meta.type))
+ throw common::RuntimeException(
+ common::stringFormat("Type mismatch for list element of property '{}'.", name));
+
+ auto& cell = rows.at(row_idx).props[idx];
+ if (std::holds_alternative<std::monostate>(cell)) {
+ cell = std::vector<Scalar>{}; // initialize list
+ }
+ auto& vec = std::get<std::vector<Scalar>>(cell);
+ vec.push_back(scalar);
+ }
+
+ // Return number of rows currently stored.
+ size_t RowCount() const noexcept { return rows.size(); }
+
+ // Return a const reference to the schema vector.
+ const std::vector<PropMeta>& Schema() const noexcept { return schema; }
+
+ // Return a const reference to an entire row. Throws if index OOB.
+ const WriteRow& GetRow(size_t row_idx) const {
+ if (row_idx >= rows.size())
+ throw common::RuntimeException{
+ common::stringFormat("row index {} out of range.", row_idx)};
+ return rows.at(row_idx);
+ }
+
+ // Return rows
+ const std::vector<WriteRow>& GetRows() const noexcept { return rows; }
+
+ // Return a scalar property by row index and property name.
+ // If the property is absent or null, return std::nullopt.
+ // If the property exists but is not SINGLE, throw.
+ std::optional<Scalar> GetScalarProperty(size_t row_idx, const std::string& name) const {
+ size_t idx = idxOf(name);
+ const auto& meta = schema[idx];
+ if (meta.card != Cardinality::SINGLE)
+ throw common::RuntimeException{
+ common::stringFormat("property '{}' is not SINGLE.", name)};
+ const auto& cell = rows.at(row_idx).props[idx];
+ if (std::holds_alternative<std::monostate>(cell))
+ return std::nullopt;
+ if (!std::holds_alternative<Scalar>(cell))
+ throw common::RuntimeException{
+ common::stringFormat("internal data error: '{}' not SINGLE.", name)};
+ return std::get<Scalar>(cell);
+ }
+
+ // Return a list property by row index and property name.
+ // If the property is absent or null, returns empty vector (not nullopt).
+ // If the property exists but is not LIST, throw.
+ std::vector<Scalar> GetListProperty(size_t row_idx, const std::string& name) const {
+ size_t idx = idxOf(name);
+ const auto& meta = schema[idx];
+ if (meta.card != Cardinality::LIST)
+ throw common::RuntimeException{
+ common::stringFormat("property '{}' is not LIST.", name)};
+ const auto& cell = rows.at(row_idx).props[idx];
+ if (std::holds_alternative<std::monostate>(cell))
+ return {};
+ if (!std::holds_alternative<std::vector<Scalar>>(cell))
+ throw common::RuntimeException{
+ common::stringFormat("internal data error: '{}' not LIST.", name)};
+ return std::get<std::vector<Scalar>>(cell);
+ }
+
+ // Return whether a given property in a row is unset/null.
+ bool IsNullProperty(size_t row_idx, const std::string& name) const {
+ size_t idx = idxOf(name);
+ const auto& cell = rows.at(row_idx).props[idx];
+ return std::holds_alternative<std::monostate>(cell);
+ }
+
+private:
+ std::vector<PropMeta> schema;
+ case_insensitive_map_t<size_t> name_to_idx;
+ std::vector<WriteRow> rows;
+
+ // Resolve property name to schema index. Throws if not found.
+ size_t idxOf(const std::string& name) const {
+ auto it = name_to_idx.find(name);
+ if (it == name_to_idx.end())
+ throw common::RuntimeException{common::stringFormat("No such property: {}.", name)};
+ return it->second;
+ }
+};
+
+struct GrapharExportFunction {
+ static constexpr const char* name = "COPY_GRAPHAR";
+
+ static function_set getFunctionSet();
+};
+
+enum class GrapharWriterFormat { CSV, PARQUET /*, ORC*/, ALL };
+
+struct GrapharExportOptions {
+ std::shared_ptr<WriterOptions> wopt;
+
+ std::shared_ptr<WriterOptions::CSVOptionBuilder> csvOptionBuilder;
+
+ std::shared_ptr<WriterOptions::ParquetOptionBuilder> parquetOptionBuilder;
+
+ std::shared_ptr<WriterOptions::ORCOptionBuilder> orcOptionBuilder;
+
+ explicit GrapharExportOptions() = delete;
+
+ explicit GrapharExportOptions(case_insensitive_map_t<common::Value> parsingOptions);
+
+ // CSV write options
+ void setCSVNullString(common::Value& value);
+
+ void setCSVDelimiter(common::Value& value);
+
+ void setCSVIncludeHeader(common::Value& value);
+
+ void setCSVBatchSize(common::Value& value);
+
+ void setCSVEOL(common::Value& value);
+
+ void setCSVQuotingStyle(common::Value& value);
+
+ // Parquet write options
+ void setParquetDictionaryPageSizeLimit(common::Value& value);
+
+ void setParquetWriteBatchSize(common::Value& value);
+
+ void setParquetMaxRowGroupLength(common::Value& value);
+
+ void setParquetDataPageSize(common::Value& value);
+
+ void setParquetMaxStatisticsSize(common::Value& value);
+
+ void setParquetCompressionLevel(common::Value& value);
+
+ void setParquetDataPageVersion(common::Value& value);
+
+ void setParquetVersion(common::Value& value);
+
+ void setParquetEncoding(common::Value& value);
+
+ void setParquetCompression(common::Value& value);
+
+ void setParquetCoerceTimestamps(common::Value& value);
+
+ void setParquetEnableDictionary(common::Value& value);
+
+ void setParquetEnableStatistics(common::Value& value);
+
+ void setParquetEnableStoreDecimalAsInteger(common::Value& value);
+
+ void setParquetEnableWritePageIndex(common::Value& value);
+
+ void setParquetCompliantNestedTypes(common::Value& value);
+
+ void setParquetUseThreads(common::Value& value);
+
+ void setParquetEnableDeprecatedInt96Timestamps(common::Value& value);
+
+ void setParquetAllowTruncatedTimestamps(common::Value& value);
+
+ void setParquetStoreSchema(common::Value& value);
+};
+
+struct ExportGrapharBindData : public ExportFuncBindData {
+ GrapharExportOptions exportOptions;
+ std::shared_ptr<GraphInfo> graphInfo;
+ std::string tableName;
+ std::string targetDir;
+ ValidateLevel validateLevel;
+
+ ExportGrapharBindData(std::vector<std::string> columnNames, const std::string& fileName,
+ GrapharExportOptions grapharWriterOptions, std::string tableName, std::string targetDir,
+ ValidateLevel validateLevel);
+
+ std::unique_ptr<ExportFuncBindData> copy() const override {
+ return std::make_unique<ExportGrapharBindData>(columnNames, fileName, exportOptions,
+ tableName, targetDir, validateLevel);
+ }
+};
+
+struct ExportGrapharSharedState : public ExportFuncSharedState {
+ // mutex for thread safety in combineFunc.
+ std::mutex mtx;
+
+ // vertex info and builder.
+ std::shared_ptr<VertexInfo> vertexInfo;
+ std::shared_ptr<builder::VerticesBuilder> verticesBuilder;
+
+ // edge info and builder.
+ std::shared_ptr<EdgeInfo> edgeInfo;
+ std::shared_ptr<builder::EdgesBuilder> edgesBuilder;
+
+ // determine if export vertex or edge.
+ bool is_edge = false;
+
+ void init(main::ClientContext& context, const ExportFuncBindData& bindData) override;
+};
+
+struct ExportGrapharLocalState : public ExportFuncLocalState {
+ std::shared_ptr<WriteRowsBuffer> buffer = nullptr;
+
+ ExportGrapharLocalState() = default;
+
+ explicit ExportGrapharLocalState(const std::vector<PropMeta>& schema)
+ : buffer(std::make_shared<WriteRowsBuffer>(schema)) {}
+};
+
+std::unique_ptr<ExportFuncBindData> bindFunc(ExportFuncBindInput& bindInput);
+
+void initSharedState(ExportFuncSharedState& sharedState, main::ClientContext& context,
+ const ExportFuncBindData& bindData);
+
+std::shared_ptr<ExportFuncSharedState> createSharedStateFunc();
+
+std::unique_ptr<ExportFuncLocalState> initLocalState(main::ClientContext&,
+ const ExportFuncBindData&, std::vector<bool>);
+
+void sinkFunc(ExportFuncSharedState&, ExportFuncLocalState& localState,
+ const ExportFuncBindData& bindData, std::vector<std::shared_ptr<ValueVector>> inputVectors);
+
+void combineFunc(ExportFuncSharedState& sharedState, ExportFuncLocalState& localState);
+
+void finalizeFunc(ExportFuncSharedState& sharedState);
+
+} // namespace graphar_extension
+} // namespace kuzu
diff --git a/cpp/extensions/kuzu-extension/src/include/function/graphar_metadata.h b/cpp/extensions/kuzu-extension/src/include/function/graphar_metadata.h
new file mode 100644
index 00000000..19d63344
--- /dev/null
+++ b/cpp/extensions/kuzu-extension/src/include/function/graphar_metadata.h
@@ -0,0 +1,91 @@
+#pragma once
+
+#include "binder/binder.h"
+#include "common/case_insensitive_map.h"
+#include "common/copy_constructors.h"
+#include "common/exception/binder.h"
+#include "common/exception/runtime.h"
+#include "common/string_utils.h"
+#include "common/types/types.h"
+#include "common/vector/value_vector.h"
+#include "function/table/bind_data.h"
+#include "function/table/bind_input.h"
+#include "function/table/scan_file_function.h"
+#include "function/table/table_function.h"
+#include "graphar/api/high_level_reader.h"
+#include "graphar/types.h"
+#include "main/client_context.h"
+#include "utils/graphar_utils.h"
+
+namespace kuzu {
+namespace graphar_extension {
+
+using namespace kuzu::function;
+using namespace kuzu::common;
+
+#define METADATA_SEPARATOR "|"
+#define GRAPH_METADATA_COLUMN_NUM 7
+#define VERTEX_METADATA_COLUMN_NUM 6
+#define EDGE_METADATA_COLUMN_NUM 11
+
+enum class GrapharMetadataType {
+ GRAPH,
+ VERTEX,
+ EDGE,
+};
+
+struct GrapharMetadataFunction {
+ static constexpr const char* name = "GRAPHAR_METADATA";
+
+ static function_set getFunctionSet();
+};
+
+struct GrapharMetadataBindData final : function::ScanFileBindData {
+ std::shared_ptr<graphar::GraphInfo> graph_info;
+ GrapharMetadataType metadata_type;
+ std::string table_name;
+ std::vector<LogicalType> column_types;
+ std::vector<std::string> column_names;
+ std::shared_ptr<graphar::VertexInfo> vertex_info;
+ std::shared_ptr<graphar::EdgeInfo> edge_info;
+
+ GrapharMetadataBindData(binder::expression_vector columns, main::ClientContext* context,
+ std::shared_ptr<graphar::GraphInfo> graph_info, GrapharMetadataType metadata_type,
+ std::string table_name, std::vector<LogicalType> column_types,
+ std::vector<std::string> column_names, std::shared_ptr<graphar::VertexInfo> vertex_info,
+ std::shared_ptr<graphar::EdgeInfo> edge_info)
+ : ScanFileBindData{std::move(columns), 0 /* numRows */, FileScanInfo{}, context},
+ graph_info{std::move(graph_info)}, metadata_type{metadata_type},
+ table_name{std::move(table_name)}, column_types{std::move(column_types)},
+ column_names{std::move(column_names)}, vertex_info{std::move(vertex_info)},
+ edge_info{std::move(edge_info)} {}
+
+ GrapharMetadataBindData(const GrapharMetadataBindData& other)
+ : ScanFileBindData{other}, graph_info{other.graph_info}, metadata_type{other.metadata_type},
+ table_name{other.table_name}, column_types{copyVector(other.column_types)},
+ column_names{other.column_names}, vertex_info{other.vertex_info},
+ edge_info{other.edge_info} {}
+
+ std::unique_ptr<TableFuncBindData> copy() const override {
+ return std::make_unique<GrapharMetadataBindData>(*this);
+ }
+};
+
+struct GrapharMetadataSharedState : public TableFuncSharedState {
+ // table func can be called once only.
+ bool finished;
+
+ explicit GrapharMetadataSharedState(bool finished) : finished(finished) {}
+};
+
+// Functions and structs exposed for use
+std::unique_ptr<function::TableFuncSharedState> initGrapharMetadataSharedState(
+ const function::TableFuncInitSharedStateInput& input);
+
+std::unique_ptr<TableFuncBindData> metadataBindFunc(main::ClientContext* context,
+ const TableFuncBindInput* input);
+
+offset_t metadataTableFunc(const TableFuncInput& input, TableFuncOutput& output);
+
+} // namespace graphar_extension
+} // namespace kuzu
diff --git a/cpp/extensions/kuzu-extension/src/include/function/graphar_scan.h b/cpp/extensions/kuzu-extension/src/include/function/graphar_scan.h
new file mode 100644
index 00000000..0192bbfc
--- /dev/null
+++ b/cpp/extensions/kuzu-extension/src/include/function/graphar_scan.h
@@ -0,0 +1,117 @@
+#pragma once
+
+#include <atomic>
+#include <functional>
+
+#include "binder/binder.h"
+#include "common/case_insensitive_map.h"
+#include "common/copy_constructors.h"
+#include "common/exception/binder.h"
+#include "common/exception/runtime.h"
+#include "common/exception/not_implemented.h"
+#include "common/string_utils.h"
+#include "common/types/types.h"
+#include "function/table/bind_data.h"
+#include "function/table/bind_input.h"
+#include "function/table/scan_file_function.h"
+#include "function/table/table_function.h"
+#include "graphar/api/high_level_reader.h"
+#include "main/client_context.h"
+#include "utils/graphar_utils.h"
+
+namespace kuzu {
+namespace graphar_extension {
+
+struct GrapharScanFunction {
+ static constexpr const char* name = "GRAPHAR_SCAN";
+
+ static function::function_set getFunctionSet();
+};
+
+// Setter for vertex iterator
+using VertexColumnSetter =
+ std::function<void(graphar::VertexIter&, function::TableFuncOutput&, kuzu::common::idx_t)>;
+
+// Setter for edge iterator
+using EdgeColumnSetter = std::function<void(graphar::EdgeIter&, function::TableFuncOutput&,
+ kuzu::common::idx_t, std::shared_ptr<graphar::VerticesCollection>)>;
+
+using column_name_idx_map_t = std::unordered_map<std::string, uint64_t>;
+
+class KuzuColumnInfo {
+public:
+ explicit KuzuColumnInfo(std::vector<std::string> columnNames);
+ DELETE_COPY_DEFAULT_MOVE(KuzuColumnInfo);
+
+ uint64_t getFieldIdx(std::string fieldName) const;
+
+private:
+ column_name_idx_map_t colNameToIdx;
+ std::vector<std::string> colNames;
+};
+
+struct GrapharScanBindData final : function::ScanFileBindData {
+ std::shared_ptr<graphar::GraphInfo> graph_info;
+ std::shared_ptr<KuzuColumnInfo> column_info;
+ std::string table_name;
+ std::vector<std::string> column_names;
+ std::vector<kuzu::common::LogicalType> column_types;
+ std::vector<VertexColumnSetter> vertex_column_setters;
+ std::vector<EdgeColumnSetter> edge_column_setters;
+ bool is_edge = false;
+ uint64_t max_threads;
+ std::unordered_map<std::string, std::string> edges_from_to_mapping;
+
+ uint64_t getFieldIdx(std::string fieldName) const {
+ return column_info->getFieldIdx(fieldName);
+ }
+
+ GrapharScanBindData(binder::expression_vector columns, common::FileScanInfo fileScanInfo,
+ main::ClientContext* context, std::shared_ptr<graphar::GraphInfo> graph_info,
+ std::string table_name, std::vector<std::string> column_names,
+ std::vector<kuzu::common::LogicalType> column_types, bool is_edge);
+
+ GrapharScanBindData(const GrapharScanBindData& other)
+ : ScanFileBindData(other), graph_info(other.graph_info), column_info(other.column_info),
+ table_name(other.table_name), column_names(other.column_names),
+ column_types(copyVector(other.column_types)),
+ vertex_column_setters(other.vertex_column_setters),
+ edge_column_setters(other.edge_column_setters), is_edge(other.is_edge),
+ max_threads(other.max_threads) {}
+
+ std::unique_ptr<TableFuncBindData> copy() const override {
+ return std::make_unique<GrapharScanBindData>(*this);
+ }
+};
+
+struct GrapharScanSharedState : public function::TableFuncSharedState {
+ graphar::Result<std::shared_ptr<graphar::VerticesCollection>> maybe_vertices_collection;
+ graphar::Result<std::shared_ptr<graphar::EdgesCollection>> maybe_edges_collection;
+ graphar::Result<std::shared_ptr<graphar::VerticesCollection>> maybe_from_vertices_collection;
+ graphar::Result<std::shared_ptr<graphar::VerticesCollection>> maybe_to_vertices_collection;
+ std::atomic<size_t> next_index{0};
+ size_t collection_count;
+ size_t batch_size;
+ std::vector<graphar::EdgeIter> edge_batch_iters; // if is_edge, store batch start iterators
+
+ GrapharScanSharedState(
+ graphar::Result<std::shared_ptr<graphar::VerticesCollection>> maybeVerticesCollection,
+ graphar::Result<std::shared_ptr<graphar::EdgesCollection>> maybeEdgesCollection,
+ graphar::Result<std::shared_ptr<graphar::VerticesCollection>>
+ maybe_from_vertices_collection,
+ graphar::Result<std::shared_ptr<graphar::VerticesCollection>> maybe_to_vertices_collection,
+ uint64_t max_threads, bool is_edge);
+};
+
+// Functions and structs exposed for use
+std::unique_ptr<function::TableFuncSharedState> initGrapharScanSharedState(
+ const function::TableFuncInitSharedStateInput& input);
+
+std::unique_ptr<function::TableFuncBindData> bindFunc(main::ClientContext* context,
+ const function::TableFuncBindInput* input);
+
+common::offset_t tableFunc(const function::TableFuncInput& input,
+ function::TableFuncOutput& output);
+
+} // namespace graphar_extension
+} // namespace kuzu
diff --git a/cpp/extensions/kuzu-extension/src/include/main/graphar_extension.h b/cpp/extensions/kuzu-extension/src/include/main/graphar_extension.h
new file mode 100644
index 00000000..feac46e9
--- /dev/null
+++ b/cpp/extensions/kuzu-extension/src/include/main/graphar_extension.h
@@ -0,0 +1,17 @@
+#pragma once
+
+#include "extension/extension.h"
+
+namespace kuzu {
+namespace graphar_extension {
+
+class GrapharExtension final : public extension::Extension {
+public:
+ static constexpr char EXTENSION_NAME[] = "GRAPHAR";
+
+public:
+ static void load(main::ClientContext* context);
+};
+
+} // namespace graphar_extension
+} // namespace kuzu
diff --git a/cpp/extensions/kuzu-extension/src/include/utils/graphar_utils.h b/cpp/extensions/kuzu-extension/src/include/utils/graphar_utils.h
new file mode 100644
index 00000000..20253f98
--- /dev/null
+++ b/cpp/extensions/kuzu-extension/src/include/utils/graphar_utils.h
@@ -0,0 +1,86 @@
+#pragma once
+
+#include "common/exception/not_implemented.h"
+#include "common/string_utils.h"
+#include "common/types/types.h"
+#include "graphar/types.h"
+
+namespace kuzu {
+namespace graphar_extension {
+
+using namespace kuzu::common;
+using namespace graphar;
+
+#define SEPARATOR '_'
+#define DEFAULT_GRAPHAR_LABEL ".graphar"
+#define DEFAULT_TARGET_DIR "/tmp/"
+#define FROM_COL_NAME "from"
+#define TO_COL_NAME "to"
+#define INTERNAL_ID_COL_NAME "internal_id"
+#define INTERNAL_FROM_COL_NAME "internal_from"
+#define INTERNAL_TO_COL_NAME "internal_to"
+
+// Candidate names for edge endpoints
+const std::vector<std::string> srcCandidates = {"src", "source", "from", "src_id", "srcId",
+ "source_id", "internal_from", "internalFrom"};
+const std::vector<std::string> dstCandidates = {"dst", "dest", "target", "to", "dst_id", "dstId",
+ "target_id", "internal_to", "internalTo"};
+
+/**
+ * Get the first token from a string separated by '-'.
+ */
+std::string getFirstToken(const std::string& input);
+
+/*
+ * Remove the trailing ".graphar" from the file path if it exists.
+ */
+void getYamlNameWithoutGrapharLabelInPlace(std::string& filePath);
+
+/**
+ * Checks whether a string ends with a given suffix.
+ */
+bool ends_with(const std::string& s, const std::string& suffix);
+
+/**
+ * parse_is_edge
+ * - Input: A file path string (e.g., "xxx/ldbc/parquet/person.vertex.yml")
+ * - Return: true -> if the file is an edge schema (.edge.yml / .edge.yaml)
+ * false -> if the file is a vertex schema (.vertex.yml / .vertex.yaml)
+ * - Exception: Throws std::invalid_argument if the file name does not match
+ * either vertex or edge schema suffixes.
+ *
+ * This function performs case-insensitive matching by converting the file name
+ * to uppercase via common::StringUtils::getUpper. It supports both '/' and '\\'
+ * as valid path separators.
+ */
+bool parse_is_edge(const std::string& path);
+
+/**
+ * tryParseEdgeTableName
+ * - Input: A table name string that may represent an edge relationship
+ * - Output: If parsing succeeds, sets src, edge, dst references to the parsed parts
+ * - Return: true -> if the table name was successfully parsed as "src.edge.dst" format
+ * false -> if the table name does not match the expected format
+ *
+ * This function attempts to parse a table name using multiple separator character '_'.
+ * It expects exactly 3 non-empty parts in the format:
+ * [source_vertex_label][separator][edge_label][separator][destination_vertex_label]
+ *
+ * Examples of valid inputs:
+ * - "page_likes_page" -> src="page", edge="likes", dst="page"
+ */
+bool tryParseEdgeTableName(const std::string& table_name, std::string& src, std::string& edge,
+ std::string& dst);
+
+/*
+ * Convert GraphAr data type to Kuzu logical type.
+ */
+LogicalType grapharTypeToKuzuType(std::shared_ptr<graphar::DataType> type);
+
+/*
+ * Convert Kuzu logical type to GraphAr data type.
+ */
+graphar::Type kuzuTypeToGrapharType(const LogicalType& type);
+
+} // namespace graphar_extension
+} // namespace kuzu
diff --git a/cpp/extensions/kuzu-extension/src/main/CMakeLists.txt b/cpp/extensions/kuzu-extension/src/main/CMakeLists.txt
new file mode 100644
index 00000000..9c021b0e
--- /dev/null
+++ b/cpp/extensions/kuzu-extension/src/main/CMakeLists.txt
@@ -0,0 +1,7 @@
+add_library(kuzu_graphar_main
+ OBJECT
+ graphar_extension.cpp)
+
+set(GRAPHAR_EXTENSION_OBJECT_FILES
+ ${GRAPHAR_EXTENSION_OBJECT_FILES} $<TARGET_OBJECTS:kuzu_graphar_main>
+ PARENT_SCOPE)
diff --git a/cpp/extensions/kuzu-extension/src/main/graphar_extension.cpp b/cpp/extensions/kuzu-extension/src/main/graphar_extension.cpp
new file mode 100644
index 00000000..ea7eb218
--- /dev/null
+++ b/cpp/extensions/kuzu-extension/src/main/graphar_extension.cpp
@@ -0,0 +1,39 @@
+#include "main/graphar_extension.h"
+
+#include "function/graphar_export.h"
+#include "function/graphar_metadata.h"
+#include "function/graphar_scan.h"
+#include "main/client_context.h"
+#include "main/database.h"
+
+namespace kuzu {
+namespace graphar_extension {
+
+void GrapharExtension::load(main::ClientContext* context) {
+ auto& db = *context->getDatabase();
+ extension::ExtensionUtils::addTableFunc<GrapharScanFunction>(db);
+ extension::ExtensionUtils::addTableFunc<GrapharMetadataFunction>(db);
+ extension::ExtensionUtils::addScalarFunc<GrapharExportFunction>(db);
+}
+
+} // namespace graphar_extension
+} // namespace kuzu
+
+#if defined(BUILD_DYNAMIC_LOAD)
+extern "C" {
+// Because we link against the static library on windows, we implicitly inherit KUZU_STATIC_DEFINE,
+// which cancels out any exporting, so we can't use KUZU_API.
+#if defined(_WIN32)
+#define INIT_EXPORT __declspec(dllexport)
+#else
+#define INIT_EXPORT __attribute__((visibility("default")))
+#endif
+INIT_EXPORT void init(kuzu::main::ClientContext* context) {
+ kuzu::graphar_extension::GrapharExtension::load(context);
+}
+
+INIT_EXPORT const char* name() {
+ return kuzu::graphar_extension::GrapharExtension::EXTENSION_NAME;
+}
+}
+#endif
diff --git a/cpp/extensions/kuzu-extension/src/utils/CMakeLists.txt b/cpp/extensions/kuzu-extension/src/utils/CMakeLists.txt
new file mode 100644
index 00000000..9c19d62f
--- /dev/null
+++ b/cpp/extensions/kuzu-extension/src/utils/CMakeLists.txt
@@ -0,0 +1,7 @@
+add_library(kuzu_graphar_utils
+ OBJECT
+ graphar_utils.cpp)
+
+set(GRAPHAR_EXTENSION_OBJECT_FILES
+ ${GRAPHAR_EXTENSION_OBJECT_FILES} $<TARGET_OBJECTS:kuzu_graphar_utils>
+ PARENT_SCOPE)
diff --git a/cpp/extensions/kuzu-extension/src/utils/graphar_utils.cpp b/cpp/extensions/kuzu-extension/src/utils/graphar_utils.cpp
new file mode 100644
index 00000000..7b210326
--- /dev/null
+++ b/cpp/extensions/kuzu-extension/src/utils/graphar_utils.cpp
@@ -0,0 +1,159 @@
+#include "utils/graphar_utils.h"
+
+namespace kuzu {
+namespace graphar_extension {
+
+std::string getFirstToken(const std::string& input) {
+ size_t pos = input.find('-');
+ if (pos == std::string::npos) {
+ return input;
+ }
+ return input.substr(0, pos);
+}
+
+void getYamlNameWithoutGrapharLabelInPlace(std::string& filePath) {
+ const std::string grapharLabel = DEFAULT_GRAPHAR_LABEL;
+ if (ends_with(filePath, grapharLabel)) {
+ // remove the trailing ".graphar" in-place
+ filePath.erase(filePath.size() - grapharLabel.size());
+ }
+}
+
+bool ends_with(const std::string& filePath, const std::string& suffix) {
+ if (filePath.size() < suffix.size())
+ return false;
+ return filePath.compare(filePath.size() - suffix.size(), suffix.size(), suffix) == 0;
+}
+
+bool parse_is_edge(const std::string& path) {
+ if (path.empty()) {
+ throw std::invalid_argument("empty path");
+ }
+
+ // Extract the file name (substring after the last '/' or '\\')
+ const size_t pos = path.find_last_of("/\\");
+ const std::string filename = (pos == std::string::npos) ? path : path.substr(pos + 1);
+
+ if (filename.empty()) {
+ throw std::invalid_argument("path has no filename");
+ }
+
+ // Convert to uppercase for case-insensitive comparison
+ const std::string upperFilename = common::StringUtils::getUpper(filename);
+
+ // Uppercase suffix constants
+ constexpr const char* EDGE_YAML = ".EDGE.YAML";
+ constexpr const char* EDGE_YML = ".EDGE.YML";
+ constexpr const char* VERTEX_YAML = ".VERTEX.YAML";
+ constexpr const char* VERTEX_YML = ".VERTEX.YML";
+
+ if (ends_with(upperFilename, EDGE_YAML) || ends_with(upperFilename, EDGE_YML)) {
+ return true;
+ }
+ if (ends_with(upperFilename, VERTEX_YAML) || ends_with(upperFilename, VERTEX_YML)) {
+ return false;
+ }
+
+ throw std::invalid_argument(
+ "filename does not end with .vertex.(yml|yaml) or .edge.(yml|yaml)");
+}
+
+bool tryParseEdgeTableName(const std::string& table_name, std::string& src,
+ std::string& edge, std::string& dst) {
+ // Split by '_'.
+ std::vector<std::string> parts;
+ size_t start = 0;
+ for (size_t i = 0; i <= table_name.size(); ++i) {
+ if (i == table_name.size() || table_name[i] == SEPARATOR) {
+ parts.push_back(table_name.substr(start, i - start));
+ start = i + 1;
+ }
+ }
+ if (parts.size() == 3 && !parts[0].empty() && !parts[1].empty() && !parts[2].empty()) {
+ src = parts[0];
+ edge = parts[1];
+ dst = parts[2];
+ return true;
+ }
+ return false;
+}
+
+LogicalType grapharTypeToKuzuType(std::shared_ptr<graphar::DataType> type) {
+ graphar::Type type_id = type->id();
+ switch (type_id) {
+ case graphar::Type::BOOL:
+ return LogicalType::BOOL();
+ case graphar::Type::INT64:
+ return LogicalType::INT64();
+ case graphar::Type::INT32:
+ return LogicalType::INT32();
+ case graphar::Type::FLOAT:
+ return LogicalType::FLOAT();
+ case graphar::Type::STRING:
+ return LogicalType::STRING();
+ case graphar::Type::DOUBLE:
+ return LogicalType::DOUBLE();
+ case graphar::Type::DATE:
+ return LogicalType::DATE();
+ case graphar::Type::TIMESTAMP:
+ return LogicalType::TIMESTAMP();
+ case graphar::Type::LIST: {
+ auto value_type_id = type->value_type()->id();
+ switch (value_type_id) {
+ case graphar::Type::BOOL:
+ return LogicalType::LIST(LogicalType::BOOL());
+ case graphar::Type::INT64:
+ return LogicalType::LIST(LogicalType::INT64());
+ case graphar::Type::INT32:
+ return LogicalType::LIST(LogicalType::INT32());
+ case graphar::Type::FLOAT:
+ return LogicalType::LIST(LogicalType::FLOAT());
+ case graphar::Type::STRING:
+ return LogicalType::LIST(LogicalType::STRING());
+ case graphar::Type::DOUBLE:
+ return LogicalType::LIST(LogicalType::DOUBLE());
+ case graphar::Type::DATE:
+ return LogicalType::LIST(LogicalType::DATE());
+ case graphar::Type::TIMESTAMP:
+ return LogicalType::LIST(LogicalType::TIMESTAMP());
+ default:
+ throw NotImplementedException{"GraphAr's List Type with value type " +
+ std::to_string(static_cast<int>(value_type_id)) +
+ " is not implemented."};
+ }
+ }
+ default:
+ throw NotImplementedException{
+ "GraphAr's Type " + std::to_string(static_cast<int>(type_id)) + " is not implemented."};
+ }
+}
+
+graphar::Type kuzuTypeToGrapharType(const LogicalType& type) {
+ switch (type.getLogicalTypeID()) {
+ case LogicalTypeID::BOOL:
+ return graphar::Type::BOOL;
+ case LogicalTypeID::INT64:
+ return graphar::Type::INT64;
+ case LogicalTypeID::INT32:
+ return graphar::Type::INT32;
+ case LogicalTypeID::FLOAT:
+ return graphar::Type::FLOAT;
+ case LogicalTypeID::STRING:
+ return graphar::Type::STRING;
+ case LogicalTypeID::DOUBLE:
+ return graphar::Type::DOUBLE;
+ case LogicalTypeID::DATE:
+ return graphar::Type::DATE;
+ case LogicalTypeID::TIMESTAMP:
+ return graphar::Type::TIMESTAMP;
+ case LogicalTypeID::LIST:
+ return graphar::Type::LIST;
+ default:
+ throw NotImplementedException{"Kuzu's Type " +
+ std::to_string(static_cast<int>(type.getLogicalTypeID())) +
+ " is not implemented."};
+ }
+}
+
+} // namespace graphar_extension
+} // namespace kuzu
diff --git a/cpp/extensions/kuzu-extension/test/README.md b/cpp/extensions/kuzu-extension/test/README.md
new file mode 100644
index 00000000..8fabaac3
--- /dev/null
+++ b/cpp/extensions/kuzu-extension/test/README.md
@@ -0,0 +1,19 @@
+## Test cases/examples
+
+```shell
+# test for copy clause
+g++ graphar_extension_copy_test.cpp -std=c++20 -ggdb -lkuzu -lpthread -o graphar_extension_copy_test
+./graphar_extension_copy_test
+
+# test for load clause
+g++ graphar_extension_load_test.cpp -std=c++20 -ggdb -lkuzu -lpthread -o graphar_extension_load_test
+./graphar_extension_load_test
+
+# test for copy to clause
+g++ graphar_extension_copy_to_test.cpp -std=c++20 -ggdb -lkuzu -lpthread -o graphar_extension_copy_to_test
+./graphar_extension_copy_to_test
+
+# test for metadata
+g++ graphar_extension_metadata_test.cpp -std=c++20 -ggdb -lkuzu -lpthread -o graphar_extension_metadata_test
+./graphar_extension_metadata_test
+```
\ No newline at end of file
diff --git a/cpp/extensions/kuzu-extension/test/graphar_extension_copy_test.cpp b/cpp/extensions/kuzu-extension/test/graphar_extension_copy_test.cpp
new file mode 100644
index 00000000..24c28ed1
--- /dev/null
+++ b/cpp/extensions/kuzu-extension/test/graphar_extension_copy_test.cpp
@@ -0,0 +1,121 @@
+#include <iostream>
+#include <filesystem>
+#include "kuzu.hpp"
+
+using namespace kuzu::main;
+
+int main()
+{
+ // Create an empty on-disk database and connect to it
+ SystemConfig systemConfig(
+ /*bufferPoolSize=*/-1u,
+ /*maxNumThreads=*/16,
+ /*enableCompression=*/true,
+ /*readOnly=*/false);
+ auto database = std::make_unique<Database>("test", systemConfig);
+
+ // Connect to the database.
+ auto connection = std::make_unique<Connection>(database.get());
+
+ // auto loadResult = connection->query("LOAD graphar;");
+ auto loadResult = connection->query("LOAD EXTENSION \"xxx/libgraphar.kuzu_extension\";");
+ if (!loadResult->isSuccess())
+ {
+ std::cerr << loadResult->getErrorMessage() << std::endl;
+ std::filesystem::remove_all("test");
+ return -1;
+ }
+
+ // Create the schema.
+ auto createTableResult = connection->query("CREATE NODE TABLE Person(id INT64, firstName STRING, lastName STRING, gender STRING, PRIMARY KEY (id))");
+ if (!createTableResult->isSuccess())
+ {
+ std::cerr << createTableResult->getErrorMessage() << std::endl;
+ std::filesystem::remove_all("test");
+ return -1;
+ }
+
+ // Copy explain
+ auto copyExplainResult = connection->query("EXPLAIN COPY Person FROM \"xxx/ldbc_sample.graph.yml\" (file_format=\"graphar\", table_name=\"person\")");
+ // auto copyExplainResult = connection->query("EXPLAIN COPY Person FROM \"xxx/LdbcSample.graph.yml\" (file_format=\"graphar\", table_name=\"Person\")");
+ if (!copyExplainResult->isSuccess())
+ {
+ std::cerr << copyExplainResult->getErrorMessage() << std::endl;
+ std::filesystem::remove_all("test");
+ return -1;
+ }
+
+ while (copyExplainResult->hasNext())
+ {
+ auto row = copyExplainResult->getNext();
+ std::cout << row->getValue(0)->getValue<std::string>() << std::endl;
+ }
+
+ // Load data.
+ auto copyResult = connection->query("COPY Person FROM \"xxx/ldbc_sample.graph.yml\" (file_format=\"graphar\", table_name=\"person\")");
+ // auto copyResult = connection->query("COPY Person FROM \"xxx/LdbcSample.graph.yml\" (file_format=\"graphar\", table_name=\"Person\")");
+ if (!copyResult->isSuccess())
+ {
+ std::cerr << copyResult->getErrorMessage() << std::endl;
+ std::filesystem::remove_all("test");
+ return -1;
+ }
+
+ // Execute a simple query.
+ auto result =
+ connection->query("MATCH (p:Person) RETURN p.id, p.firstName, p.lastName, p.gender;");
+
+ // Output query result.
+ while (result->hasNext())
+ {
+ auto row = result->getNext();
+ std::cout << row->getValue(0)->getValue<int64_t>() << " "
+ << row->getValue(1)->getValue<std::string>() << " "
+ << row->getValue(2)->getValue<std::string>() << " "
+ << row->getValue(3)->getValue<std::string>() << std::endl;
+ }
+
+ /* EDGE */
+ connection->query("CREATE REL TABLE KNOWS(FROM Person TO Person, creationDate STRING)");
+ // connection->query("CREATE REL TABLE KNOWS(FROM Person TO Person, creationDate TIMESTAMP)");
+ // connection->query("CREATE REL TABLE KNOWS(FROM Person TO Person, creationDate DATE)");
+
+ // Copy edge explain
+ // auto copyEdgeExplainResult = connection->query("EXPLAIN COPY KNOWS FROM \"xxx/ldbc_sample_timestamp.graph.yml\" (file_format=\"graphar\", table_name=\"person_knows-timestamp_person\")");
+ // auto copyEdgeExplainResult = connection->query("EXPLAIN COPY KNOWS FROM \"xxx/ldbc_sample_date.graph.yml\" (file_format=\"graphar\", table_name=\"person_knows-date_person\")");
+ auto copyEdgeExplainResult = connection->query("EXPLAIN COPY KNOWS FROM \"xxx/ldbc_sample.graph.yml\" (file_format=\"graphar\", table_name=\"person_knows_person\")");
+ // auto copyEdgeExplainResult = connection->query("EXPLAIN COPY KNOWS FROM \"xxx/LdbcSample.graph.yml\" (file_format=\"graphar\", table_name=\"Person_Knows_Person\")");
+ if (!copyEdgeExplainResult->isSuccess())
+ {
+ std::cerr << copyEdgeExplainResult->getErrorMessage() << std::endl;
+ std::filesystem::remove_all("test");
+ return -1;
+ }
+
+ // auto copyEdgeResult = connection->query("COPY KNOWS FROM \"xxx/ldbc_sample_timestamp.graph.yml\" (file_format=\"graphar\", table_name=\"person_knows-timestamp_person\")");
+ // auto copyEdgeResult = connection->query("COPY KNOWS FROM \"xxx/ldbc_sample_date.graph.yml\" (file_format=\"graphar\", table_name=\"person_knows-date_person\")");
+ auto copyEdgeResult = connection->query("COPY KNOWS FROM \"xxx/ldbc_sample.graph.yml\" (file_format=\"graphar\", table_name=\"person_knows_person\")");
+ // auto copyEdgeResult = connection->query("COPY KNOWS FROM \"xxx/LdbcSample.graph.yml\" (file_format=\"graphar\", table_name=\"Person_Knows_Person\")");
+ if (!copyEdgeResult->isSuccess())
+ {
+ std::cerr << copyEdgeResult->getErrorMessage() << std::endl;
+ std::filesystem::remove_all("test");
+ return -1;
+ }
+
+ auto edgeResult =
+ connection->query("MATCH (a:Person)-[k:KNOWS]->(b:Person) RETURN a.id, k.creationDate, b.id;");
+ while (edgeResult->hasNext())
+ {
+ auto row = edgeResult->getNext();
+ std::cout << row->getValue(0)->getValue<int64_t>() << " "
+ << row->getValue(1)->getValue<std::string>() << " "
+ // << row->getValue(1)->getValue<int64_t>() << " "
+ // << row->getValue(1)->getValue<int32_t>() << " "
+ << row->getValue(2)->getValue<int64_t>() << std::endl;
+ }
+
+ std::filesystem::remove_all("test");
+
+ return 0;
+}
\ No newline at end of file
diff --git a/cpp/extensions/kuzu-extension/test/graphar_extension_copy_to_test.cpp b/cpp/extensions/kuzu-extension/test/graphar_extension_copy_to_test.cpp
new file mode 100644
index 00000000..b0c10e88
--- /dev/null
+++ b/cpp/extensions/kuzu-extension/test/graphar_extension_copy_to_test.cpp
@@ -0,0 +1,139 @@
+#include <iostream>
+#include <filesystem>
+#include "kuzu.hpp"
+
+using namespace kuzu::main;
+
+int main()
+{
+ // Create an empty on-disk database and connect to it
+ SystemConfig systemConfig(
+ /*bufferPoolSize=*/-1u,
+ /*maxNumThreads=*/16,
+ /*enableCompression=*/true,
+ /*readOnly=*/false);
+ auto database = std::make_unique<Database>("test", systemConfig);
+
+ // Connect to the database.
+ auto connection = std::make_unique<Connection>(database.get());
+
+ // auto loadResult = connection->query("LOAD graphar;");
+ auto loadResult = connection->query("LOAD EXTENSION \"xxx/libgraphar.kuzu_extension\";");
+ if (!loadResult->isSuccess())
+ {
+ std::cerr << loadResult->getErrorMessage() << std::endl;
+ std::filesystem::remove_all("test");
+ return -1;
+ }
+
+ // Create the schema.
+ connection->query("CREATE NODE TABLE Person(internal_id INT64, id INT64, firstName STRING, lastName STRING, gender STRING, PRIMARY KEY (id))");
+
+ // Copy explain
+ auto copyExplainResult = connection->query("EXPLAIN COPY Person FROM \"xxx/ldbc_sample.graph.yml\" (file_format=\"graphar\", table_name=\"person\")");
+ while (copyExplainResult->hasNext())
+ {
+ auto row = copyExplainResult->getNext();
+ std::cout << row->getValue(0)->getValue<std::string>() << std::endl;
+ }
+
+ // Load data.
+ auto copyResult = connection->query("COPY Person FROM \"xxx/ldbc_sample.graph.yml\" (file_format=\"graphar\", table_name=\"person\")");
+ if (!copyResult->isSuccess())
+ {
+ std::cerr << copyResult->getErrorMessage() << std::endl;
+ std::filesystem::remove_all("test");
+ return -1;
+ }
+
+ // Execute a simple query.
+ auto result =
+ connection->query("MATCH (p:Person) RETURN p.internal_id, p.id, p.firstName, p.lastName, p.gender ORDER BY p.internal_id ASC LIMIT 10;");
+ // connection->query("MATCH (p:Person) RETURN p.internal_id, p.id, p.firstName, p.lastName, p.gender LIMIT 10;");
+ while (result->hasNext())
+ {
+ auto row = result->getNext();
+ std::cout << row->getValue(0)->getValue<int64_t>() << " "
+ << row->getValue(1)->getValue<int64_t>() << " "
+ << row->getValue(2)->getValue<std::string>() << " "
+ << row->getValue(3)->getValue<std::string>() << " "
+ << row->getValue(4)->getValue<std::string>() << std::endl;
+ }
+
+ // export to gar data.
+ auto exportPlan = connection->query("EXPLAIN COPY (MATCH (p:Person) RETURN p.id as id, p.firstName as firstName, p.lastName as lastName, p.gender as gender ORDER BY p.internal_id ASC) TO \"xxx/ldbc_sample.graph.yml.graphar\" (table_name=\"person\")");
+ // auto exportPlan = connection->query("EXPLAIN COPY (MATCH (p:Person) RETURN p.id as id, p.firstName as firstName, p.lastName as lastName, p.gender as gender) TO \"xxx/ldbc_sample.graph.yml.graphar\" (table_name=\"person\")");
+ if (!exportPlan->isSuccess())
+ {
+ std::cerr << exportPlan->getErrorMessage() << std::endl;
+ std::filesystem::remove_all("test");
+ return -1;
+ }
+
+ while (exportPlan->hasNext())
+ {
+ auto row = exportPlan->getNext();
+ std::cout << row->getValue(0)->getValue<std::string>() << std::endl;
+ }
+
+ auto exportResult = connection->query("COPY (MATCH (p:Person) RETURN p.id as id, p.firstName as firstName, p.lastName as lastName, p.gender as gender ORDER BY p.internal_id ASC) TO \"xxx/ldbc_sample.graph.yml.graphar\" (table_name=\"person\", target_dir=\"xxx/copyToTestSet/\")");
+ // auto exportResult = connection->query("COPY (MATCH (p:Person) RETURN p.id as id, p.firstName as firstName, p.lastName as lastName, p.gender as gender) TO \"xxx/ldbc_sample.graph.yml.graphar\" (table_name=\"person\", target_dir=\"xxx/copyToTestSet/\")");
+ if (!exportResult->isSuccess())
+ {
+ std::cerr << exportResult->getErrorMessage() << std::endl;
+ std::filesystem::remove_all("test");
+ return -1;
+ }
+ std::cout << "Vertex exported successfully." << std::endl;
+
+ /* EDGE */
+ connection->query("CREATE REL TABLE KNOWS(FROM Person TO Person, creationDate STRING)");
+
+ auto copyEdgeResult = connection->query("COPY KNOWS FROM \"xxx/ldbc_sample.graph.yml\" (file_format=\"graphar\", table_name=\"person_knows_person\")");
+ if (!copyEdgeResult->isSuccess())
+ {
+ std::cerr << copyEdgeResult->getErrorMessage() << std::endl;
+ std::filesystem::remove_all("test");
+ return -1;
+ }
+
+ auto edgeResult =
+ connection->query("MATCH (a:Person)-[k:KNOWS]->(b:Person) RETURN a.id AS from, a.internal_id AS internal_from, k.creationDate AS creationDate, b.id AS to, b.internal_id AS internal_to ORDER BY internal_from ASC, internal_to ASC LIMIT 10;");
+ while (edgeResult->hasNext())
+ {
+ auto row = edgeResult->getNext();
+ std::cout << row->getValue(0)->getValue<int64_t>() << " "
+ << row->getValue(1)->getValue<int64_t>() << " "
+ << row->getValue(2)->getValue<std::string>() << " "
+ << row->getValue(3)->getValue<int64_t>() << " "
+ << row->getValue(4)->getValue<int64_t>() << std::endl;
+ }
+
+ // export edge to gar data.
+ auto exportEdgePlan = connection->query("EXPLAIN COPY (MATCH (a:Person)-[k:KNOWS]->(b:Person) RETURN a.internal_id as from, k.creationDate as creationDate, b.internal_id as to) TO \"xxx/ldbc_sample.graph.yml.graphar\" (table_name=\"person_knows_person\", target_dir=\"xxx/copyToTestSet/\")");
+ if (!exportEdgePlan->isSuccess())
+ {
+ std::cerr << exportEdgePlan->getErrorMessage() << std::endl;
+ std::filesystem::remove_all("test");
+ return -1;
+ }
+
+ while (exportEdgePlan->hasNext())
+ {
+ auto row = exportEdgePlan->getNext();
+ std::cout << row->getValue(0)->getValue<std::string>() << std::endl;
+ }
+
+ auto exportEdgeResult = connection->query("COPY (MATCH (a:Person)-[k:KNOWS]->(b:Person) RETURN a.internal_id as from, k.creationDate as creationDate, b.internal_id as to) TO \"xxx/ldbc_sample.graph.yml.graphar\" (table_name=\"person_knows_person\", target_dir=\"xxx/copyToTestSet/\")");
+ if (!exportEdgeResult->isSuccess())
+ {
+ std::cerr << exportEdgeResult->getErrorMessage() << std::endl;
+ std::filesystem::remove_all("test");
+ return -1;
+ }
+
+ std::cout << "Edge exported successfully." << std::endl;
+
+ std::filesystem::remove_all("test");
+ return 0;
+}
\ No newline at end of file
diff --git a/cpp/extensions/kuzu-extension/test/graphar_extension_load_test.cpp b/cpp/extensions/kuzu-extension/test/graphar_extension_load_test.cpp
new file mode 100644
index 00000000..1ebf8be0
--- /dev/null
+++ b/cpp/extensions/kuzu-extension/test/graphar_extension_load_test.cpp
@@ -0,0 +1,66 @@
+#include <iostream>
+#include <filesystem>
+#include "kuzu.hpp"
+
+using namespace kuzu::main;
+
+int main()
+{
+ // Create an empty on-disk database and connect to it
+ SystemConfig systemConfig(
+ /*bufferPoolSize=*/-1u,
+ /*maxNumThreads=*/16,
+ /*enableCompression=*/true,
+ /*readOnly=*/false);
+ auto database = std::make_unique<Database>("test", systemConfig);
+
+ // Connect to the database.
+ auto connection = std::make_unique<Connection>(database.get());
+
+ // auto loadResult = connection->query("LOAD graphar;");
+ auto result = connection->query("LOAD EXTENSION \"xxx/libgraphar.kuzu_extension\";");
+ if (!result->isSuccess())
+ {
+ std::cerr << result->getErrorMessage() << std::endl;
+ std::filesystem::remove_all("test");
+ return -1;
+ }
+
+ // Load explain
+ auto loadExplainResult = connection->query("EXPLAIN LOAD FROM \"xxx/ldbc_sample.graph.yml\" (file_format=\"graphar\", table_name=\"person\") RETURN *");
+ if (!loadExplainResult->isSuccess())
+ {
+ std::cerr << loadExplainResult->getErrorMessage() << std::endl;
+ std::filesystem::remove_all("test");
+ return -1;
+ }
+
+ while (loadExplainResult->hasNext())
+ {
+ auto row = loadExplainResult->getNext();
+ std::cout << row->getValue(0)->getValue<std::string>() << std::endl;
+ }
+
+ // Load data.
+ auto loadResult = connection->query("LOAD FROM \"xxx/ldbc_sample.graph.yml\" (file_format=\"graphar\", table_name=\"person\") RETURN *");
+ if (!loadResult->isSuccess())
+ {
+ std::cerr << loadResult->getErrorMessage() << std::endl;
+ std::filesystem::remove_all("test");
+ return -1;
+ }
+
+ while (loadResult->hasNext())
+ {
+ auto row = loadResult->getNext();
+ std::cout << row->getValue(0)->getValue<int64_t>() << " "
+ << row->getValue(1)->getValue<int64_t>() << " "
+ << row->getValue(2)->getValue<std::string>() << " "
+ << row->getValue(3)->getValue<std::string>() << " "
+ << row->getValue(4)->getValue<std::string>() << std::endl;
+ }
+
+ std::filesystem::remove_all("test");
+
+ return 0;
+}
\ No newline at end of file
diff --git a/cpp/extensions/kuzu-extension/test/graphar_extension_metadata_test.cpp b/cpp/extensions/kuzu-extension/test/graphar_extension_metadata_test.cpp
new file mode 100644
index 00000000..ec5676b6
--- /dev/null
+++ b/cpp/extensions/kuzu-extension/test/graphar_extension_metadata_test.cpp
@@ -0,0 +1,88 @@
+#include <iostream>
+#include <filesystem>
+#include <memory>
+#include "kuzu.hpp"
+
+using namespace kuzu::main;
+using namespace kuzu::common;
+using namespace kuzu::processor;
+
+void printRow(const std::unique_ptr<QueryResult> &result, const std::shared_ptr<FlatTuple> &row, std::ostream &os) {
+ auto colNames = result->getColumnNames();
+ for (uint32_t i = 0; i < result->getNumColumns(); i++) {
+ std::string colName = colNames[i];
+ auto val = row->getValue(i);
+
+ os << colName << ": ";
+ os << val->toString();
+ os << "\n";
+ }
+ os << "--------------------------\n";
+}
+
+
+int main() {
+ SystemConfig cfg(
+ /*bufferPoolSize=*/-1u,
+ /*maxNumThreads=*/16,
+ /*enableCompression=*/true,
+ /*readOnly=*/false);
+ auto db = std::make_unique<Database>("test", cfg);
+ auto conn = std::make_unique<Connection>(db.get());
+
+ auto loadRes = conn->query("LOAD EXTENSION \"xxx/libgraphar.kuzu_extension\";");
+ if (!loadRes->isSuccess()) {
+ std::cerr << "Load extension failed: " << loadRes->getErrorMessage() << std::endl;
+ std::filesystem::remove_all("test");
+ return -1;
+ }
+ std::cout << "Extension loaded.\n";
+
+ // === GRAPH metadata ===
+ {
+ auto result = conn->query("CALL GRAPHAR_METADATA('xxx/modern_graph.graph.yml') RETURN *;");
+ if (!result->isSuccess()) {
+ std::cerr << "GRAPHAR_METADATA error: " << result->getErrorMessage() << std::endl;
+ std::filesystem::remove_all("test");
+ return -1;
+ }
+ std::cout << "[GRAPH Metadata]\n";
+ while (result->hasNext()) {
+ auto row = result->getNext();
+ printRow(result, row, std::cout);
+ }
+ }
+
+ // === VERTEX metadata ===
+ {
+ auto result = conn->query("CALL GRAPHAR_METADATA('xxx/modern_graph.graph.yml', vertex_type := 'person') RETURN *;");
+ if (!result->isSuccess()) {
+ std::cerr << "VERTEX Metadata error: " << result->getErrorMessage() << std::endl;
+ std::filesystem::remove_all("test");
+ return -1;
+ }
+ std::cout << "[VERTEX Metadata]\n";
+ while (result->hasNext()) {
+ auto row = result->getNext();
+ printRow(result, row, std::cout);
+ }
+ }
+
+ // === EDGE metadata ===
+ {
+ auto result = conn->query("CALL GRAPHAR_METADATA('xxx/modern_graph.graph.yml', edge_type := 'person_knows_person') RETURN *;");
+ if (!result->isSuccess()) {
+ std::cerr << "EDGE Metadata error: " << result->getErrorMessage() << std::endl;
+ std::filesystem::remove_all("test");
+ return -1;
+ }
+ std::cout << "[EDGE Metadata]\n";
+ while (result->hasNext()) {
+ auto row = result->getNext();
+ printRow(result, row, std::cout);
+ }
+ }
+
+ std::filesystem::remove_all("test");
+ return 0;
+}
diff --git a/testing b/testing
deleted file mode 160000
index 12b4b175..00000000
--- a/testing
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 12b4b17561ca3e414366b176a8760b7ee825f7d9
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@graphar.apache.org
For additional commands, e-mail: commits-help@graphar.apache.org