You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by li...@apache.org on 2026/08/05 14:03:06 UTC
(seatunnel) branch dev updated: [Feature][Connector-V2] Add HugeGraph source connector and refactor sink with multi-mapping support (#11413)
This is an automated email from the ASF dual-hosted git repository.
davidzollo pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 543f2a5a64 [Feature][Connector-V2] Add HugeGraph source connector and refactor sink with multi-mapping support (#11413)
543f2a5a64 is described below
commit 543f2a5a64d9aac7c63760aa5051ecaf4b852107
Author: kkPig <li...@gmail.com>
AuthorDate: Wed Aug 5 22:02:59 2026 +0800
[Feature][Connector-V2] Add HugeGraph source connector and refactor sink with multi-mapping support (#11413)
Co-authored-by: ljx <17...@163.com>
Co-authored-by: Claude Opus 4.7 <no...@anthropic.com>
Co-authored-by: kl1l <Ka...@newegg.com>
Co-authored-by: Cursor <cu...@cursor.com>
---
.../en/connectors/changelog/connector-hugegraph.md | 6 +
docs/en/connectors/sink/HugeGraph.md | 152 +++-
docs/en/connectors/source/HugeGraph.md | 190 +++++
.../zh/connectors/changelog/connector-hugegraph.md | 13 +
docs/zh/connectors/sink/HugeGraph.md | 158 ++--
docs/zh/connectors/source/HugeGraph.md | 190 +++++
plugin-mapping.properties | 1 +
pom.xml | 2 +-
.../connector-hugegraph/pom.xml | 107 ++-
.../seatunnel/hugegraph/buffer/BatchBuffer.java | 375 +++++++-
.../hugegraph/buffer/GraphElementEnvelope.java | 79 ++
.../hugegraph/client/HugeGraphClient.java | 862 ++++++++++++++++++-
.../hugegraph/client/HugeGraphOperations.java | 79 ++
.../MappingConfig.java => client/PageResult.java} | 16 +-
.../config/HugeGraphConnectionConfig.java | 132 +++
...ppingConfig.java => HugeGraphDataSaveMode.java} | 30 +-
.../hugegraph/config/HugeGraphOptions.java | 70 +-
...ingConfig.java => HugeGraphSchemaSaveMode.java} | 24 +-
.../hugegraph/config/HugeGraphSinkConfig.java | 229 ++++-
.../hugegraph/config/HugeGraphSinkOptions.java | 67 +-
.../hugegraph/config/HugeGraphSourceConfig.java | 156 ++++
.../hugegraph/config/HugeGraphSourceOptions.java | 92 ++
.../seatunnel/hugegraph/config/LabelOptions.java | 57 ++
.../seatunnel/hugegraph/config/ListFormat.java | 51 ++
.../seatunnel/hugegraph/config/MappingConfig.java | 191 ++++-
.../hugegraph/config/ReservedColumns.java | 69 ++
.../seatunnel/hugegraph/config/SchemaConfig.java | 11 +-
.../exception/HugeGraphConnectorErrorCode.java | 1 +
.../seatunnel/hugegraph/mapper/EdgeMapper.java | 432 +++++++---
.../hugegraph/mapper/GraphDataMapper.java | 30 +-
.../seatunnel/hugegraph/mapper/VertexMapper.java | 341 ++++++--
.../hugegraph/sink/HugeGraphSaveModeHandler.java | 296 +++++++
.../seatunnel/hugegraph/sink/HugeGraphSink.java | 28 +-
.../hugegraph/sink/HugeGraphSinkFactory.java | 42 +-
.../hugegraph/sink/HugeGraphSinkWriter.java | 769 +++++++++++++++--
.../hugegraph/source/HugeGraphSource.java | 98 +++
.../hugegraph/source/HugeGraphSourceFactory.java | 340 ++++++++
.../hugegraph/source/HugeGraphSourceReader.java | 669 +++++++++++++++
.../hugegraph/source/HugeGraphSourceSplit.java | 172 ++++
.../source/HugeGraphSourceSplitEnumerator.java | 274 ++++++
.../hugegraph/source/HugeGraphSourceState.java | 51 ++
.../hugegraph/source/LabelTableContext.java | 69 ++
.../seatunnel/hugegraph/utils/DataTypeUtil.java | 181 +++-
.../hugegraph/utils/HugeGraphTypeConverter.java | 99 +++
.../seatunnel/hugegraph/utils/SchemaManager.java | 555 ++++++++++++
.../seatunnel/hugegraph/utils/SchemaValidator.java | 578 +++++++++++--
.../hugegraph/buffer/BatchBufferTest.java | 284 +++++++
.../hugegraph/client/HugeGraphClientTest.java | 181 ++++
.../config/HugeGraphConnectionConfigTest.java | 147 ++++
.../hugegraph/config/HugeGraphSinkConfigTest.java | 501 +++++++----
.../config/HugeGraphSourceConfigTest.java | 171 ++++
.../seatunnel/hugegraph/mapper/EdgeMapperTest.java | 154 ++++
.../hugegraph/mapper/VertexMapperTest.java | 309 +++++++
.../sink/HugeGraphSaveModeHandlerTest.java | 223 +++++
.../hugegraph/sink/HugeGraphSinkFactoryTest.java | 54 ++
.../sink/HugeGraphSinkWriterMultiTableTest.java | 203 +++++
.../sink/HugeGraphSinkWriterUpdateTest.java | 435 ++++++++++
.../source/HugeGraphSourceFactoryTest.java | 297 +++++++
.../source/HugeGraphSourceReaderTest.java | 840 ++++++++++++++++++
.../source/HugeGraphSourceSplitEnumeratorTest.java | 406 +++++++++
.../hugegraph/utils/DataTypeUtilTest.java | 116 +++
.../utils/HugeGraphTypeConverterTest.java | 73 ++
.../hugegraph/utils/SchemaManagerNullableTest.java | 160 ++++
.../hugegraph/utils/SchemaValidatorTest.java | 319 +++++++
.../connector-hugegraph-e2e/pom.xml | 28 +
.../e2e/connector/hugegraph/HugeGraphIT.java | 941 ++++++++++-----------
.../e2e/connector/hugegraph/HugeGraphSourceIT.java | 371 ++++++++
.../hugegraph/fake_to_hugegraph_edge.conf | 63 ++
.../hugegraph/fake_to_hugegraph_multi_mapping.conf | 76 ++
...fake_to_hugegraph_multi_mapping_edge_first.conf | 77 ++
.../hugegraph/fake_to_hugegraph_vertex.conf | 52 ++
.../hugegraph/hugegraph_byte_object_to_assert.conf | 108 +++
.../hugegraph/hugegraph_edge_to_assert.conf | 126 +++
.../hugegraph_vertex_checkpoint_to_assert.conf | 82 ++
.../hugegraph/hugegraph_vertex_to_assert.conf | 100 +++
75 files changed, 14066 insertions(+), 1265 deletions(-)
diff --git a/docs/en/connectors/changelog/connector-hugegraph.md b/docs/en/connectors/changelog/connector-hugegraph.md
index 5bdc416dcf..26b40ba456 100644
--- a/docs/en/connectors/changelog/connector-hugegraph.md
+++ b/docs/en/connectors/changelog/connector-hugegraph.md
@@ -1,7 +1,13 @@
+---
+title: HugeGraph
+---
+
<details><summary> Change Log </summary>
| Change | Commit | Version |
| --- | --- |---------|
+|[Fix][Connector-V2] Enforce topology-safe execution order in HugeGraph multi-mapping sink|https://github.com/apache/seatunnel/commit/467980a6a0| dev |
+|[Feature][Connector-V2] Add HugeGraph source connector and refactor sink with multi-mapping support|https://github.com/apache/seatunnel/commit/0f503024d7| dev |
|[Feature][Connector-V2] Support sink connector for Apache HugeGraph|https://github.com/apache/seatunnel/pull/10002/commits/002a653d11f48c3f76b47db23f5f2a68bc9d690c| 2.3.12 |
</details>
diff --git a/docs/en/connectors/sink/HugeGraph.md b/docs/en/connectors/sink/HugeGraph.md
index 9f24c73a0e..e8ac7a39c1 100644
--- a/docs/en/connectors/sink/HugeGraph.md
+++ b/docs/en/connectors/sink/HugeGraph.md
@@ -15,14 +15,14 @@ This connector supports writing data as vertices or edges, providing flexible ma
- [x] [batch](../../introduction/concepts/connector-v2-features.md)
- [ ] [exactly-once](../../introduction/concepts/connector-v2-features.md)
- [ ] [cdc](../../introduction/concepts/connector-v2-features.md)
-- [ ] [support multiple table write](../../introduction/concepts/connector-v2-features.md)
+- [x] [support multiple table write](../../introduction/concepts/connector-v2-features.md)
- [x] [timer flush](../../introduction/concepts/connector-v2-features.md)
The connector writes rows as either vertices or edges. It supports insert, update, and delete row kinds, and it can flush buffered records by `batch_size` or `batch_interval_ms`.
:::caution
-The HugeGraph schema must already exist before the job writes data. Create the required property keys, vertex labels, and edge labels in HugeGraph first, then use `schema_config` to map SeaTunnel fields to that existing graph schema.
+New `mappings` configurations default to `schema_save_mode = CREATE_SCHEMA_WHEN_NOT_EXIST` and create missing HugeGraph PropertyKey/VertexLabel/EdgeLabel definitions before writing. Legacy `schema_config` jobs retain the previous `ERROR_WHEN_SCHEMA_NOT_EXIST` behavior unless this option is explicitly set.
:::
@@ -32,28 +32,72 @@ The HugeGraph schema must already exist before the job writes data. Create the r
| ------------------- | ------- | -------- | ------------- |--------------------------------------------------------------------------------|
| `host` | String | Yes | - | The host of the HugeGraph server. |
| `port` | Integer | Yes | - | The port of the HugeGraph server. |
+| `protocol` | String | No | `http` | Server protocol: `http` or `https`. HTTPS uses the JVM trust store. |
| `graph_name` | String | Yes | - | The name of the graph to write to. |
-| `graph_space` | String | No | - | The graph space of the graph to be operated on. |
+| `graph_space` | String | No | `DEFAULT` | The graph space the graph belongs to. |
| `username` | String | No | - | The username for HugeGraph authentication. |
| `password` | String | No | - | The password for HugeGraph authentication. |
| `batch_size` | Integer | No | 500 | The number of records to buffer before writing to HugeGraph in a single batch. |
| `batch_interval_ms` | Integer | No | 5000 | The maximum time in milliseconds to wait before flushing a batch. |
-| `max_retries` | Integer | No | 3 | The maximum number of times to retry a failed write operation. |
-| `retry_backoff_ms` | Integer | No | 5000 | The backoff time between retries in milliseconds. |
+| `batch_failure_fallback` | Boolean | No | true | When a batch insert fails, fall back to inserting the batch record by record so a single bad ("poison") record no longer fails the whole batch. Failed records are logged and skipped; the rest succeed. If every record fails (systemic error), it is surfaced. Set to `false` to fail the whole batch instead. |
+| `max_insert_errors` | Integer | No | 500 | Maximum number of records that may be skipped by the single-record fallback (`batch_failure_fallback=true`) before the task is failed. Bounds the otherwise unlimited silent skipping of poison records. Set to `-1` for unlimited. Only applies when `batch_failure_fallback` is enabled. |
+| `failure_data_path` | String | No | - | Optional local directory. When set, every record skipped by the single-record fallback is appended (mapped id, label, properties and the server error) to a per-subtask file (`hugegraph-sink-failures-subtask-N.log`) for offline investigation. In cluster mode the file is created on the worker node running the sink subtask. |
+| `check_vertex` | Boolean | No | false | Whether the server verifies that an edge's source/target vertices exist when writing edges. When `false`, edges whose endpoints were never loaded are written as orphan edges (or trigger server-side phantom vertex auto-creation). Enable to reject such edges. |
+| `max_retries` | Integer | No | 3 | Retries after the initial attempt. Set to `0` to disable retries. |
+| `retry_backoff_ms` | Integer | No | 5000 | Base backoff between retries in ms. Grows exponentially per attempt (`retry_backoff_ms * 2^(attempt-1)`), capped at `retry_backoff_max_ms`. |
+| `retry_backoff_max_ms` | Integer | No | 30000 | Upper bound in ms for the exponential retry backoff. |
## Sink Options
-| Name | Type | Required | Default Value | Description |
-| ------------------ | ------ | -------- | ------------- |-----------------------------------------------------------------------------------------------------|
-| `schema_config` | Object | Yes | - | The configuration for mapping the input data to HugeGraph's schema (vertices or edges). |
-| `selected_fields` | List | No | - | A list of fields to be selected from the input data. If not specified, all fields will be used. |
-| `ignored_fields` | List | No | - | A list of fields to be ignored from the input data. Mutually exclusive with `selected_fields`. |
+| Name | Type | Required | Default Value | Description |
+|----------------------------|---------|----------|---------------|-------------|
+| `mappings` | List | Yes | - | Recommended mapping configuration. Each entry maps input rows to one HugeGraph vertex or edge label. |
+| `schema_save_mode` | Enum | No | `CREATE_SCHEMA_WHEN_NOT_EXIST` for `mappings`; `ERROR_WHEN_SCHEMA_NOT_EXIST` for legacy `schema_config` | Schema management mode. |
+| `data_save_mode` | Enum | No | `APPEND_DATA` | How pre-existing data is handled before writing. `APPEND_DATA` keeps existing data. `DROP_DATA` deletes, once at job start, only the data of the labels this job targets (edges then vertices), preserving their schema and any other labels' data; the drop is scoped per label (so one table's drop does not wipe another) and is not re-run on checkpoint restart. |
+| `delete_vertex_with_edges` | Boolean | No | `false` for `mappings`; `true` for legacy `schema_config` | When true, DELETE rows for vertices also delete associated edges. |
+| `schema_config` | Object | No | - | Deprecated legacy mapping object. Use `mappings` instead. Either `mappings` or `schema_config` must be specified. |
+| `selected_fields` | List | No | - | Deprecated. Still honored with legacy `schema_config`; use mapping `properties` for new jobs. |
+| `ignored_fields` | List | No | - | Deprecated. Still honored with legacy `schema_config`; use mapping `properties` for new jobs. |
-`selected_fields` and `ignored_fields` are applied before the row is mapped to HugeGraph. Keep every field used by `idFields`, `sourceConfig.idFields`, `targetConfig.idFields`, `mapping.fieldMapping`, or `mapping.sortKeys`; otherwise the connector cannot build the vertex or edge ID.
+If both `mappings` and `schema_config` are configured, `mappings` wins and `schema_config` is ignored with a warning.
-### Schema Configuration (`schema_config`)
+### Mapping Configuration (`mappings`)
-`schema_config` defines how one input stream is mapped to a specific vertex or edge label in HugeGraph.
+Each `mappings` entry defines how input rows are mapped to one HugeGraph vertex or edge label.
+
+| Name | Type | Required | Default Value | Description |
+| ------------------ |--------------------| ---------- | ------------- |----------------------------------------------------------------------------------------------------------|
+| `type` | String | Yes | - | The type of graph element to map to. Must be `VERTEX` or `EDGE`. |
+| `label` | String | Yes | - | The label of the vertex or edge in HugeGraph. |
+| `properties` | `List<String>` | No | - | Source field names written as HugeGraph properties. If empty, all input fields are considered. |
+| `ttl` | Long | No | - | The time-to-live for the vertex or edge in seconds. |
+| `ttlStartTime` | String | No | - | The start time for the TTL. |
+| `enableLabelIndex` | String | No | - | Reserved label-index setting passed through the mapping config. |
+| `userdata` | `Map<String, Object>` | No | - | User-defined data associated with the label. |
+| `idStrategy` | String | For Vertex | - | The ID generation strategy for vertices, such as `PRIMARY_KEY`, `CUSTOMIZE_STRING`, `CUSTOMIZE_NUMBER`, `CUSTOMIZE_UUID`, or `AUTOMATIC`. |
+| `idFields` | `List<string>` | For Vertex | - | A list of source field names used to generate the vertex ID. Required when `idStrategy` is not `AUTOMATIC`. |
+| `sourceConfig` | Object | For Edge | - | An object defining the mapping for the edge's source vertex. See `Source/Target Config` below. |
+| `targetConfig` | Object | For Edge | - | An object defining the mapping for the edge's target vertex. See `Source/Target Config` below. |
+| `frequency` | String | For Edge | - | The frequency of the edge, e.g., `SINGLE`, `MULTIPLE`. |
+| `sortKeys` | `List<String>` | For Edge | - | **Source field names** (as they appear in the input row, *before* `fieldMapping` is applied) whose values distinguish edges sharing the same source and target vertices. Required when `frequency = MULTIPLE`. Example: with `fieldMapping = {event_time: created_at}`, use `sortKeys = [event_time]`, not `[created_at]`. |
+| `fieldMapping` | `Map<String, String>` | No | - | A map where the key is the source field name and the value is the target property name in HugeGraph. |
+| `valueMapping` | `Map<String, Map<Object, Object>>` | No | - | Per-field value transform. Outer key = source field name; inner map = `originalValue -> newValue`. |
+| `ignored` | `List<String>` | No | - | Blacklist of source fields excluded from properties (implicit mode only). Mutually exclusive with `properties` (which acts as the selected whitelist). |
+| `updateStrategies` | `Map<String, String>` | No | - | Per-property merge strategy on write, keyed by target property name: `OVERRIDE`, `APPEND`, `SUM`, `UNION`, `BIGGER`, `SMALLER`, etc. When set, existing elements are merged instead of overwritten. |
+| `nullableKeys` | `List<String>` | No | - | Explicit allow-list of property keys that may be null on an auto-created label. When set, it overrides the default below (only these keys are nullable). Key properties (primary keys, `MULTIPLE`-edge sort keys) are always excluded. Mutually exclusive with `notNullableKeys`. |
+| `notNullableKeys` | `List<String>` | No | - | Opt-out list used with the default nullability. By default, when neither `nullableKeys` nor `notNullableKeys` is set, all non-key properties of an auto-created label are nullable; list here the properties that must instead be required. Mutually exclusive with `nullableKeys`. Only affects newly auto-created labels. |
+| `nullValues` | `List<String>` | No | - | A list of string values that should be treated as `null`. |
+| `dateFormat` | String | No | `yyyy-MM-dd` | The date format for parsing date strings. |
+| `extraDateFormats` | `List<String>` | No | - | Additional date patterns tried in order, after `dateFormat`, when parsing date strings — for sources that mix multiple date formats. |
+| `listFormat` | Object | No | - | How a raw string cell is parsed into SET/LIST property elements: `startSymbol` (default `[`), `endSymbol` (default `]`), `elemDelimiter` (default `,`), and `ignoredElems`. |
+| `unfold` | Boolean | No | false | (Vertex) Expand a list-valued CUSTOMIZE id cell into one vertex per element. INSERT/append-only. |
+| `unfoldSource` | Boolean | No | false | (Edge) Expand a list-valued source-endpoint id cell into multiple edges (CUSTOMIZE endpoint). INSERT/append-only. |
+| `unfoldTarget` | Boolean | No | false | (Edge) Expand a list-valued target-endpoint id cell into multiple edges (cartesian with source). INSERT/append-only. |
+| `timeZone` | String | No | Worker JVM default | The time zone for date parsing. When omitted, the worker JVM default time zone is used, matching the HugeGraph Source so a Source→Sink round-trip preserves absolute times. |
+
+### Legacy Schema Configuration (`schema_config`)
+
+`schema_config` defines how one input stream is mapped to a specific vertex or edge label in HugeGraph. It is deprecated; new jobs should use `mappings`.
| Name | Type | Required | Default Value | Description |
| ------------------ |--------------------| ---------- | ------------- |----------------------------------------------------------------------------------------------------------|
@@ -79,7 +123,14 @@ This object is used within an `EDGE` schema to define how to identify the source
| Name | Type | Required | Default Value | Description |
| ---------- | ------------ | -------- | ------------- |--------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `label` | String | Yes | - | The label of the source or target vertex. |
-| `idFields` | `List<String>` | Yes | - | A list of source field names from the input row used to construct the ID of the source/target vertex. The values will be concatenated to form the vertex ID. |
+| `idFields` | `List<String>` | Yes | - | A list of source field names from the input row used to construct the ID of the source/target vertex. The values will be concatenated to form the vertex ID. For a HugeGraph → HugeGraph clone, set this to the single reserved column that already carries the assembled endpoint id — `["~source_id"]` for `sourceConfig`, `["~target_id"]` for `targetConfig` — and the connector reuses that id directly (see *Cloning from a HugeGraph Sourc [...]
+
+### Cloning from a HugeGraph Source (reserved-id passthrough)
+
+When the input is the HugeGraph Source, each row carries reserved columns holding the already-assembled element ids (`~id` for a vertex; `~source_id`/`~target_id` for an edge's endpoints). To clone losslessly:
+
+- **Vertex** with a `CUSTOMIZE_STRING`/`CUSTOMIZE_NUMBER`/`CUSTOMIZE_UUID` id: set `idStrategy` to the matching `CUSTOMIZE_*` and `idFields = ["~id"]`; the original id is written verbatim. `PRIMARY_KEY` vertices instead reuse their key property columns (which the Source already emits), and `AUTOMATIC` ids cannot be preserved (the target server assigns new ones).
+- **Edge**: set `sourceConfig.idFields = ["~source_id"]` and `targetConfig.idFields = ["~target_id"]`. The endpoint ids are reused directly, so edges clone regardless of the endpoint vertices' id strategies. The target endpoint vertex labels must already exist (the connector will not auto-create a vertex label from a reserved id).
### Mapping Config (`mapping`)
@@ -88,12 +139,20 @@ This object provides advanced control over how fields and values are mapped to p
| Name | Type | Required | Default Value | Description |
| ----------------- |---------------------|----------| ------------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `fieldMapping` | `Map<String, String>` | No | - | A map where the key is the source field name and the value is the target property name in HugeGraph. If not specified, the source field name is used as the target property name. |
-| `valueMapping` | `Map<Object, Object>` | No | - | A map to transform specific field values. The key is the original value from the source, and the value is the new value to be written. |
-| `nullableKeys` | `List<String>` | No | - | A list of property keys that can have null values. |
+| `valueMapping` | `Map<String, Map<Object, Object>>` | No | - | Per-field value transform. The outer key is the source field name; the inner map is `originalValue -> newValue`. Scoping by field prevents one column's rule from affecting another (e.g. `gender` M->male will not rewrite `status` M). |
+| `ignored` | `List<String>` | No | - | Blacklist of source fields excluded from properties (implicit mode only). Mutually exclusive with `properties` (which acts as the selected whitelist). |
+| `updateStrategies` | `Map<String, String>` | No | - | Per-property merge strategy on write, keyed by target property name: `OVERRIDE`, `APPEND`, `SUM`, `UNION`, `BIGGER`, `SMALLER`, etc. When set, existing elements are merged instead of overwritten. |
+| `nullableKeys` | `List<String>` | No | - | Explicit allow-list of property keys that may be null on an auto-created label. Overrides the nullable-by-default behavior. Mutually exclusive with `notNullableKeys`. |
+| `notNullableKeys` | `List<String>` | No | - | Opt-out list of properties that must be required, used together with the nullable-by-default behavior. Mutually exclusive with `nullableKeys`. |
| `nullValues` | `List<String>` | No | - | A list of string values that should be treated as `null`. Any field containing one of these values will not be written. |
| `dateFormat` | String | No | `yyyy-MM-dd` | The date format for parsing date strings. |
-| `timeZone` | String | No | `GMT+8` | The time zone for date parsing. |
-| `sortKeys` | `List<String>` | For Edge | - | A list of property keys to sort edges with the same source and target vertices. |
+| `extraDateFormats`| `List<String>` | No | - | Additional date patterns tried in order, after `dateFormat`, when parsing date strings — for sources that mix multiple date formats. |
+| `listFormat` | Object | No | - | How a raw string cell is parsed into SET/LIST property elements: `startSymbol` (default `[`), `endSymbol` (default `]`), `elemDelimiter` (default `,`), and `ignoredElems`. |
+| `unfold` | Boolean | No | false | (Vertex) Expand a list-valued CUSTOMIZE id cell into one vertex per element. INSERT/append-only. |
+| `unfoldSource` | Boolean | No | false | (Edge) Expand a list-valued source-endpoint id cell into multiple edges (CUSTOMIZE endpoint). INSERT/append-only. |
+| `unfoldTarget` | Boolean | No | false | (Edge) Expand a list-valued target-endpoint id cell into multiple edges (cartesian with source). INSERT/append-only. |
+| `timeZone` | String | No | Worker JVM default | The time zone for date parsing. When omitted, the worker JVM default time zone is used. |
+| `sortKeys` | `List<String>` | For Edge | - | **Source field names** (before `fieldMapping` is applied) whose values distinguish edges sharing the same source and target vertices. Example: with `fieldMapping = {event_time: created_at}`, use `[event_time]`, not `[created_at]`. |
## Supported Types
@@ -121,14 +180,18 @@ The connector validates the SeaTunnel row schema against the existing HugeGraph
## Write Behavior Notes
-- For vertices, `idStrategy` controls how the vertex ID is built. `PRIMARY_KEY` joins all `idFields` with HugeGraph's primary-key format, `CUSTOMIZE_STRING` joins them with `:`, `CUSTOMIZE_NUMBER` expects one numeric field, and `CUSTOMIZE_UUID` expects one UUID field.
+- For vertices, `idStrategy` controls how the vertex ID is built. `PRIMARY_KEY` joins all `idFields` with HugeGraph's primary-key format, `CUSTOMIZE_STRING` joins multiple `idFields` with `:` (backslash-escaping any `:` in a value so distinct field tuples cannot collide; a single field is used verbatim), `CUSTOMIZE_NUMBER` expects one integer-valued field (a fractional value like `1.9` is rejected, not silently truncated), and `CUSTOMIZE_UUID` expects one UUID field.
- For edges, the connector reads the ID strategy from the existing source and target vertex labels in HugeGraph. `sourceConfig.idFields` and `targetConfig.idFields` must provide enough fields to rebuild those vertex IDs.
- `INSERT` writes a new vertex or edge, `UPDATE_AFTER` updates the existing graph element, and `DELETE` deletes it. Delete rows only need the fields required to build the element ID.
-- `mapping.nullValues` treats matching string values as null and skips those properties during writes.
+- `AUTOMATIC` vertex IDs support INSERT only. UPDATE and DELETE require a reconstructable ID strategy.
+- The sink is at-least-once. Replayed INSERT records with `AUTOMATIC` IDs can create duplicates after retries or checkpoint recovery.
+- Edge batches use the configured `check_vertex` (default `false`). With the default, vertices and edges may be written out of order and the graph reaches its final consistent state after all batches complete; set `check_vertex=true` to have the server reject edges whose endpoints do not yet exist.
+- `nullValues` treats matching string values as null and skips those properties during writes.
+- Time zone is configured **per mapping** via `timeZone` (there is no top-level `time_zone` option on the sink, unlike the HugeGraph Source, because date parsing is a per-mapping concern). When omitted it defaults to the worker JVM zone, matching the Source so a Source→Sink round-trip preserves absolute times.
## Usage Examples
-The examples below assume the corresponding HugeGraph schema has already been created.
+The examples below use the default `schema_save_mode = CREATE_SCHEMA_WHEN_NOT_EXIST`. If you set `schema_save_mode = ERROR_WHEN_SCHEMA_NOT_EXIST`, create the corresponding HugeGraph schema before running the job.
### 1. Writing Vertices
@@ -156,15 +219,15 @@ sink {
host = "localhost"
port = 8080
graph_name = "hugegraph"
- graph_space = "default"
- selected_fields = ["name", "age"]
- schema_config = {
- type = "VERTEX"
- label = "person"
- idStrategy = "PRIMARY_KEY"
- idFields = ["name"]
- properties = ["name", "age"]
- }
+ mappings = [
+ {
+ type = "VERTEX"
+ label = "person"
+ idStrategy = "PRIMARY_KEY"
+ idFields = ["name"]
+ properties = ["name", "age"]
+ }
+ ]
}
}
```
@@ -196,26 +259,25 @@ sink {
host = "localhost"
port = 8080
graph_name = "hugegraph"
- graph_space = "default"
- schema_config = {
- type = "EDGE"
- label = "knows"
- sourceConfig = {
- label = "person"
- idFields = ["person1_name"]
- }
- targetConfig = {
- label = "person"
- idFields = ["person2_name"]
- }
- properties = ["since"]
- mapping = {
+ mappings = [
+ {
+ type = "EDGE"
+ label = "knows"
+ sourceConfig = {
+ label = "person"
+ idFields = ["person1_name"]
+ }
+ targetConfig = {
+ label = "person"
+ idFields = ["person2_name"]
+ }
+ properties = ["since"]
fieldMapping = {
person1_name = "name"
person2_name = "name"
}
}
- }
+ ]
}
}
```
diff --git a/docs/en/connectors/source/HugeGraph.md b/docs/en/connectors/source/HugeGraph.md
new file mode 100644
index 0000000000..2059cf2027
--- /dev/null
+++ b/docs/en/connectors/source/HugeGraph.md
@@ -0,0 +1,190 @@
+import ChangeLog from '../changelog/connector-hugegraph.md';
+
+# HugeGraph Source Connector
+
+`Source: HugeGraph`
+
+## Description
+
+The HugeGraph source connector reads graph data from Apache HugeGraph through the HugeGraph REST API.
+
+It performs a bounded scan of one vertex label or one edge label — or of **all** labels of a type in a single job — and checkpoints its progress so a job can resume after failover.
+
+- At `parallelism = 1` it pages the label via the server-side list API, following HugeGraph page markers until the server returns `page = null`. Server-side `filter` (property-equality) is applied in this mode.
+- At `parallelism > 1` it splits the keyspace into shards (via the HugeGraph `traverser().vertexShards / edgeShards` API) and scans them across parallel readers. Because the shard scan is by key range and returns all labels, the connector filters to the configured `label` client-side. See [Parallel read](#parallel-read).
+- When `label` is omitted, it reads every label of `label_type` (default `VERTEX`) in one job, producing one output table per label. See [Read all labels](#read-all-labels).
+
+## Key Features
+
+- [x] [batch](../../introduction/concepts/connector-v2-features.md)
+- [x] [parallelism](../../introduction/concepts/connector-v2-features.md)
+- [ ] [cdc](../../introduction/concepts/connector-v2-features.md)
+
+## Options
+
+| Name | Type | Required | Default | Description |
+|--------------------|---------|----------|----------|-------------|
+| `host` | String | Yes | - | HugeGraph server host. |
+| `port` | Integer | Yes | - | HugeGraph server port. |
+| `protocol` | String | No | `http` | Server protocol: `http` or `https`. HTTPS uses the JVM trust store. |
+| `graph_name` | String | Yes | - | HugeGraph graph name. |
+| `label` | String | No | - | Vertex label or edge label to read. **When omitted, the connector reads all labels of `label_type` in one job, producing one table per label** (see [Read all labels](#read-all-labels)); `schema` and `filter` are not allowed in that mode. |
+| `schema` | Object | No | - | Output property columns declared with `schema.fields`. Reserved graph columns are added by the connector. **When omitted, the connector auto-discovers all property columns of `label` from the server (types inferred, columns ordered by name).** See [Schema auto-discovery](#schema-auto-discovery). |
+| `label_type` | Enum | No | `VERTEX` | Label type. Supported values: `VERTEX`, `EDGE`. |
+| `page_size` | Integer | No | `1000` | Number of records per HugeGraph page. Must be in range `[100, 10000]`. |
+| `split_size` | Long | No | `1048576` | Target size in bytes of each key-range shard when `parallelism > 1`. A larger value yields fewer, bigger shards. Must be at least `1048576` (1 MiB, the HugeGraph minimum shard size) — a smaller value is rejected at startup to avoid shard explosion. Ignored at `parallelism = 1`. Requires a scan-capable backend (RocksDB / HBase / Cassandra). |
+| `filter` | Map | No | - | Optional property equality conditions applied server-side, for example `{ country = "US", active = "true" }`. Only elements whose properties match all entries are returned. Every key must be a property of `label` (an unknown key fails at startup), and each value is coerced to that property's type (e.g. `"true"` → boolean, `"7"` → the numeric type) so it matches server-side — a value that cannot be coerced fails at startup instead of [...]
+| `time_zone` | String | No | Worker JVM default | ZoneId used to convert HugeGraph DATE values the server returns as an epoch/Date, for example `UTC` or `Asia/Shanghai`. It does not apply to DATE values the server already serializes as a wall-clock string (those carry no zone and are kept verbatim). Set it explicitly when workers may use different JVM time zones. |
+| `graph_space` | String | No | `DEFAULT` | The graph space the graph belongs to. |
+| `username` | String | No | - | HugeGraph username. |
+| `password` | String | No | - | HugeGraph password. |
+| `max_retries` | Integer | No | `3` | Retries after the initial attempt. Set to `0` to disable retries. |
+| `retry_backoff_ms` | Integer | No | `5000` | Base backoff between retries in ms. Grows exponentially per attempt (`retry_backoff_ms * 2^(attempt-1)`), capped at `retry_backoff_max_ms`. |
+| `retry_backoff_max_ms` | Integer | No | `30000` | Upper bound in ms for the exponential retry backoff. |
+
+## Output Schema
+
+Vertex output columns:
+
+```text
+~id, ~label, <schema.fields columns...>
+```
+
+Edge output columns:
+
+```text
+~id, ~label, ~source_id, ~source_label, ~target_id, ~target_label, <schema.fields columns...>
+```
+
+Columns prefixed with `~` are reserved columns added by the connector. HugeGraph property keys cannot start with `~`, so they do not conflict with user properties.
+
+## Type Mapping
+
+`schema.fields` must match the HugeGraph property key type. The connector validates this before reading.
+
+| HugeGraph type | SeaTunnel type |
+|----------------|----------------|
+| `TEXT` | `STRING` |
+| `BYTE` | `TINYINT` |
+| `INT` | `INT` |
+| `LONG` | `BIGINT` |
+| `FLOAT` | `FLOAT` |
+| `DOUBLE` | `DOUBLE` |
+| `BOOLEAN` | `BOOLEAN` |
+| `DATE` | `TIMESTAMP` |
+| `UUID` | `STRING` |
+| `OBJECT` | `STRING` |
+| `BLOB` | `BYTES` |
+
+### Multi-valued (LIST / SET) properties
+
+A HugeGraph property whose cardinality is `LIST` or `SET` is read as a SeaTunnel `ARRAY`. Declare it in `schema.fields` as `array<T>`, where `T` is the SeaTunnel type of the element (from the table above). For example, a `LIST<TEXT>` property named `tags` is declared as `tags = "array<string>"`.
+
+Notes:
+
+- `SET` elements have no guaranteed order on the server; use `LIST` when order matters.
+- If a property has cardinality `LIST`/`SET` on the server but is declared as a scalar (or vice versa), the job fails at startup with a message telling you the correct declaration.
+- `BLOB` elements inside a `LIST`/`SET` are not supported.
+
+## Example
+
+```hocon
+source {
+ HugeGraph {
+ host = "localhost"
+ port = 8080
+ graph_name = "hugegraph"
+ label = "person"
+ label_type = "VERTEX"
+ page_size = 1000
+ schema = {
+ fields = {
+ name = "string"
+ age = "int"
+ }
+ }
+ }
+}
+```
+
+## Schema auto-discovery
+
+`schema` is optional. When omitted, the connector connects to the server at job build time, reads the definition of `label`, and produces one output column per property key (types from the [Type Mapping](#type-mapping) table, `LIST`/`SET` as `array<T>`), ordered by property name. This is convenient for a full-label dump when you do not want to hand-declare every field.
+
+```hocon
+source {
+ HugeGraph {
+ host = "localhost"
+ port = 8080
+ graph_name = "hugegraph"
+ label = "person"
+ label_type = "VERTEX"
+ # no schema block: all properties of "person" are read
+ }
+}
+```
+
+Notes:
+
+- The label must already exist on the server, otherwise the job fails at build time.
+- A label with no property keys produces only the reserved columns (`~id`, `~label`, …).
+- Declare `schema.fields` explicitly when you want to read only a subset of properties, fix the column order, or pin the types.
+
+## Read all labels
+
+Omit `label` to read **every** label of `label_type` (default `VERTEX`) in a single job — convenient for a full-graph migration or backup instead of configuring one source per label. At job build time the connector lists all labels of the type from the server schema and produces one output table per label, each with its own auto-discovered columns (see [Schema auto-discovery](#schema-auto-discovery)). Each output row carries its label's table id, so a downstream multi-table sink routes i [...]
+
+```hocon
+source {
+ HugeGraph {
+ host = "localhost"
+ port = 8080
+ graph_name = "hugegraph"
+ label_type = "VERTEX"
+ # no label: every vertex label is read, one table each
+ }
+}
+```
+
+Notes:
+
+- One job reads vertices **or** edges, not both: set `label_type = "EDGE"` to read all edge labels.
+- `schema` is not allowed (a single schema cannot describe multiple labels) — columns are always auto-discovered per label.
+- `filter` is not allowed (a property-equality filter assumes the property exists on every label).
+- Each label becomes one `LABEL_LIST` split, distributed across readers (parallelism is bounded by the number of labels). Shard-level parallelism within a single label is not used in this mode.
+- The job fails at build time if the graph has no label of the requested type.
+
+## Parallel read
+
+For large graphs, set `parallelism > 1` to read a label in parallel. The enumerator asks HugeGraph to split the label's keyspace into shards of roughly `split_size` bytes and distributes them round-robin across readers, so throughput scales with parallelism instead of being bound by a single paging cursor.
+
+```hocon
+source {
+ HugeGraph {
+ host = "localhost"
+ port = 8080
+ graph_name = "hugegraph"
+ label = "person"
+ label_type = "VERTEX"
+ parallelism = 8
+ split_size = 1048576
+ schema = {
+ fields = {
+ name = "string"
+ age = "int"
+ }
+ }
+ }
+}
+```
+
+Notes:
+
+- Shard scans require a scan-capable backend (RocksDB / HBase / Cassandra). The `memory` backend does not support shard splitting; use `parallelism = 1` there.
+- A shard scan returns elements of all labels in the key range; the connector keeps only the configured `label`. On a graph where the target label is a small fraction of the data, a single-parallelism `filter`ed read may move less data even though it does not parallelize.
+- `filter` is not supported with `parallelism > 1`; keep `parallelism = 1` to use a server-side filter, or drop the filter to read in parallel.
+- Tune `split_size`: a smaller value yields more, smaller shards (finer load balancing, more requests); a larger value yields fewer, bigger shards. The minimum is `1048576` (1 MiB); smaller values are rejected to avoid splitting the keyspace into an excessive number of shards.
+
+## Changelog
+
+<ChangeLog />
diff --git a/docs/zh/connectors/changelog/connector-hugegraph.md b/docs/zh/connectors/changelog/connector-hugegraph.md
index e69de29bb2..26b40ba456 100644
--- a/docs/zh/connectors/changelog/connector-hugegraph.md
+++ b/docs/zh/connectors/changelog/connector-hugegraph.md
@@ -0,0 +1,13 @@
+---
+title: HugeGraph
+---
+
+<details><summary> Change Log </summary>
+
+| Change | Commit | Version |
+| --- | --- |---------|
+|[Fix][Connector-V2] Enforce topology-safe execution order in HugeGraph multi-mapping sink|https://github.com/apache/seatunnel/commit/467980a6a0| dev |
+|[Feature][Connector-V2] Add HugeGraph source connector and refactor sink with multi-mapping support|https://github.com/apache/seatunnel/commit/0f503024d7| dev |
+|[Feature][Connector-V2] Support sink connector for Apache HugeGraph|https://github.com/apache/seatunnel/pull/10002/commits/002a653d11f48c3f76b47db23f5f2a68bc9d690c| 2.3.12 |
+
+</details>
diff --git a/docs/zh/connectors/sink/HugeGraph.md b/docs/zh/connectors/sink/HugeGraph.md
index 203f50b622..7f5e94c22c 100644
--- a/docs/zh/connectors/sink/HugeGraph.md
+++ b/docs/zh/connectors/sink/HugeGraph.md
@@ -15,14 +15,14 @@ HugeGraph sink连接器允许您将数据从SeaTunnel写入Apache HugeGraph,
- [x] [批处理](../../introduction/concepts/connector-v2-features.md)
- [ ] [精确一次](../../introduction/concepts/connector-v2-features.md)
- [ ] [CDC](../../introduction/concepts/connector-v2-features.md)
-- [ ] [支持多表写入](../../introduction/concepts/connector-v2-features.md)
+- [x] [支持多表写入](../../introduction/concepts/connector-v2-features.md)
- [x] [定时刷新](../../introduction/concepts/connector-v2-features.md)
该连接器可以把输入行写成顶点或边,支持插入、更新、删除,并且可以按 `batch_size` 或 `batch_interval_ms` 刷新缓存数据。
:::caution
-运行任务前,需要先在 HugeGraph 中创建好对应的属性键、顶点标签和边标签。`schema_config` 只负责把 SeaTunnel 字段映射到已有图结构,不会自动创建 HugeGraph Schema。
+新的 `mappings` 配置默认使用 `schema_save_mode = CREATE_SCHEMA_WHEN_NOT_EXIST`,写入前会创建缺失的 HugeGraph PropertyKey/VertexLabel/EdgeLabel。Legacy `schema_config` 任务在未显式设置该选项时保留原有的 `ERROR_WHEN_SCHEMA_NOT_EXIST` 行为。
:::
@@ -32,28 +32,72 @@ HugeGraph sink连接器允许您将数据从SeaTunnel写入Apache HugeGraph,
| ------------------- | ------- | -------- | ------ | ---------------------------------------------------------------------- |
| `host` | String | 是 | - | HugeGraph服务器的主机。 |
| `port` | Integer | 是 | - | HugeGraph服务器的端口。 |
+| `protocol` | String | 否 | `http` | 服务协议,支持 `http`、`https`。HTTPS 使用 JVM trust store。 |
| `graph_name` | String | 是 | - | 要写入的图的名称。 |
-| `graph_space` | String | 否 | - | 要操作的图的图空间。 |
+| `graph_space` | String | 否 | `DEFAULT` | 图所属的图空间(graph space)。 |
| `username` | String | 否 | - | 用于HugeGraph身份验证的用户名。 |
| `password` | String | 否 | - | 用于HugeGraph身份验证的密码。 |
| `batch_size` | Integer | 否 | 500 | 在单批次写入HugeGraph之前缓冲的记录数。 |
| `batch_interval_ms` | Integer | 否 | 5000 | 刷新批次前等待的最大时间(毫秒)。 |
-| `max_retries` | Integer | 否 | 3 | 重试失败写入操作的最大次数。 |
-| `retry_backoff_ms` | Integer | 否 | 5000 | 重试之间的退避时间(毫秒)。 |
+| `batch_failure_fallback` | Boolean | 否 | true | 批量写入失败时,降级为逐条写入,使单条“毒药”记录不再拖垮整批。失败记录会记录日志并跳过,其余成功;若整批全部失败(系统性错误)则抛出。设为 `false` 则整批失败。 |
+| `max_insert_errors` | Integer | 否 | 500 | 逐条降级(`batch_failure_fallback=true`)累计跳过的失败记录达到该数量后使任务失败,用于约束原本无上限的“毒药”记录静默跳过。设为 `-1` 表示不限。仅在开启 `batch_failure_fallback` 时生效。 |
+| `failure_data_path` | String | 否 | - | 可选本地目录。设置后,逐条降级跳过的每条记录(映射后的 id、label、属性及服务端错误)会追加写入按子任务区分的文件(`hugegraph-sink-failures-subtask-N.log`)以便离线排查。集群模式下文件写在运行该 sink 子任务的 worker 节点上。 |
+| `check_vertex` | Boolean | 否 | false | 写入边时服务端是否校验边的源/目标顶点是否存在。为 `false` 时,端点从未写入的边会被写成孤儿边(或触发服务端幻影顶点自动创建)。开启后此类边会被拒绝。 |
+| `max_retries` | Integer | 否 | 3 | 首次请求失败后的重试次数。设置为 `0` 可禁用重试。 |
+| `retry_backoff_ms` | Integer | 否 | 5000 | 重试的基础退避时间(毫秒),按尝试次数指数增长(`retry_backoff_ms * 2^(attempt-1)`),上限为 `retry_backoff_max_ms`。 |
+| `retry_backoff_max_ms` | Integer | 否 | 30000 | 指数退避的上限(毫秒)。 |
## Sink选项
-| 名称 | 类型 | 是否必须 | 默认值 | 描述 |
-| ------------------ | ------ | -------- | ------ | -------------------------------------------------------------------- |
-| `schema_config` | Object | 是 | - | 将输入数据映射到HugeGraph的Schema(顶点或边)的配置。 |
-| `selected_fields` | List | 否 | - | 要从输入数据中选择的字段列表。如果未指定,将使用所有字段。 |
-| `ignored_fields` | List | 否 | - | 要从输入数据中忽略的字段列表。与`selected_fields`互斥。 |
-
-`selected_fields` 和 `ignored_fields` 会在字段映射到 HugeGraph 之前生效。请保留 `idFields`、`sourceConfig.idFields`、`targetConfig.idFields`、`mapping.fieldMapping` 或 `mapping.sortKeys` 会用到的字段,否则连接器无法生成顶点或边的 ID。
-
-### Schema配置 (`schema_config`)
-
-`schema_config` 定义一个输入流如何映射到 HugeGraph 中的某个顶点标签或边标签。
+| 名称 | 类型 | 是否必须 | 默认值 | 描述 |
+|----------------------------|---------|----------|--------|------|
+| `mappings` | List | 是 | - | 推荐的映射配置。每个条目将输入行映射到一个 HugeGraph 顶点或边标签。 |
+| `schema_save_mode` | Enum | 否 | `mappings` 为 `CREATE_SCHEMA_WHEN_NOT_EXIST`;legacy 为 `ERROR_WHEN_SCHEMA_NOT_EXIST` | Schema 管理模式。 |
+| `data_save_mode` | Enum | 否 | `APPEND_DATA` | 写入前如何处理已有数据。`APPEND_DATA` 保留已有数据;`DROP_DATA` 在任务开始时**仅**删除本任务涉及的 label 的数据(先边后点),保留其 schema 以及其他 label 的数据;删除按 label 隔离(某张表的 DROP 不会波及其他表),且在 checkpoint 重启时不会重复执行。 |
+| `delete_vertex_with_edges` | Boolean | 否 | `mappings` 为 `false`;legacy 为 `true` | 为 true 时,顶点 DELETE 行会同时删除关联边。 |
+| `schema_config` | Object | 否 | - | 已废弃的 legacy 映射对象。请使用 `mappings`。必须配置 `mappings` 或 `schema_config` 之一。 |
+| `selected_fields` | List | 否 | - | 已废弃。Legacy `schema_config` 仍会应用;新任务请使用 mapping 内的 `properties`。 |
+| `ignored_fields` | List | 否 | - | 已废弃。Legacy `schema_config` 仍会应用;新任务请使用 mapping 内的 `properties`。 |
+
+如果同时配置 `mappings` 和 `schema_config`,connector 会使用 `mappings`,并输出警告说明 `schema_config` 被忽略。
+
+### 映射配置 (`mappings`)
+
+每个 `mappings` 条目定义输入行如何映射到一个 HugeGraph 顶点标签或边标签。
+
+| 名称 | 类型 | 是否必须 | 默认值 | 描述 |
+|--------------------|---------------------|----------|---------|------|
+| `type` | String | 是 | - | 要映射到的图元素类型。必须是 `VERTEX` 或 `EDGE`。 |
+| `label` | String | 是 | - | HugeGraph 中顶点或边的标签。 |
+| `properties` | `List<String>` | 否 | - | 要写入 HugeGraph 属性的源字段名。为空时会考虑所有输入字段。 |
+| `ttl` | Long | 否 | - | 顶点或边的生存时间,单位秒。 |
+| `ttlStartTime` | String | 否 | - | TTL 的开始时间。 |
+| `enableLabelIndex` | String | 否 | - | 随 mapping 配置传入的预留标签索引配置。 |
+| `userdata` | `Map<String, Object>` | 否 | - | 与标签关联的用户自定义数据。 |
+| `idStrategy` | String | 对于顶点 | - | 顶点 ID 生成策略,例如 `PRIMARY_KEY`、`CUSTOMIZE_STRING`、`CUSTOMIZE_NUMBER`、`CUSTOMIZE_UUID` 或 `AUTOMATIC`。 |
+| `idFields` | `List<String>` | 对于顶点 | - | 用于生成顶点 ID 的源字段名。当 `idStrategy` 不是 `AUTOMATIC` 时必填。 |
+| `sourceConfig` | Object | 对于边 | - | 定义边的源顶点映射。请参阅下面的 `Source/Target Config`。 |
+| `targetConfig` | Object | 对于边 | - | 定义边的目标顶点映射。请参阅下面的 `Source/Target Config`。 |
+| `frequency` | String | 对于边 | - | 边频率,例如 `SINGLE`、`MULTIPLE`。 |
+| `sortKeys` | `List<String>` | 对于边 | - | **输入行中的源字段名**(映射前、即 `fieldMapping` 应用之前的名字),用于区分相同源点和目标点之间的多条边。当 `frequency = MULTIPLE` 时必填。示例:当 `fieldMapping = {event_time: created_at}` 时,应填 `sortKeys = [event_time]`,而不是 `[created_at]`。 |
+| `fieldMapping` | `Map<String, String>` | 否 | - | 字段映射,key 为源字段名,value 为 HugeGraph 目标属性名。 |
+| `valueMapping` | `Map<String, Map<Object, Object>>` | 否 | - | 按字段的值转换映射。外层键为源字段名,内层为 `原始值 -> 新值`。按字段隔离可避免一个列的规则影响其他列(如 `gender` 的 M->male 不会改写 `status` 的 M)。 |
+| `ignored` | `List<String>` | 否 | - | 从属性中排除的源字段黑名单(仅隐式模式生效)。与 `properties`(充当 selected 白名单)互斥。 |
+| `updateStrategies` | `Map<String, String>` | 否 | - | 写入时按目标属性名指定的属性级合并策略:`OVERRIDE`、`APPEND`、`SUM`、`UNION`、`BIGGER`、`SMALLER` 等。设置后对已存在元素做合并而非覆盖。 |
+| `nullableKeys` | `List<String>` | 否 | - | 自动建 label 时允许为 null 的属性键白名单。设置后覆盖下述默认行为(仅这些键可空)。主键、`MULTIPLE` 边的 sortKeys 等 key 属性始终排除。与 `notNullableKeys` 互斥。 |
+| `notNullableKeys` | `List<String>` | 否 | - | 与默认可空行为配合使用的反向 opt-out 列表。默认情况下(既未配 `nullableKeys` 也未配 `notNullableKeys`),自动建 label 的所有非 key 属性均可空;在此列出必须为非空的属性。与 `nullableKeys` 互斥。仅影响新建 label。 |
+| `nullValues` | `List<String>` | 否 | - | 应被视为 `null` 的字符串值列表。 |
+| `dateFormat` | String | 否 | `yyyy-MM-dd` | 用于解析日期字符串的日期格式。 |
+| `extraDateFormats` | `List<String>` | 否 | - | 解析日期字符串时,在 `dateFormat` 之后按顺序尝试的额外日期格式——用于多源汇入、日期格式不一致的场景。 |
+| `listFormat` | Object | 否 | - | 原始字符串如何解析为 SET/LIST 属性元素:`startSymbol`(默认 `[`)、`endSymbol`(默认 `]`)、`elemDelimiter`(默认 `,`)、`ignoredElems`。 |
+| `unfold` | Boolean | 否 | false | (顶点)把 list 型 CUSTOMIZE id 单元格展开为每个元素一个顶点。仅 INSERT/append。 |
+| `unfoldSource` | Boolean | 否 | false | (边)把 list 型源端点 id 单元格展开为多条边(CUSTOMIZE 端点)。仅 INSERT/append。 |
+| `unfoldTarget` | Boolean | 否 | false | (边)把 list 型目标端点 id 单元格展开为多条边(与源端笛卡尔积)。仅 INSERT/append。 |
+| `timeZone` | String | 否 | Worker JVM 默认 | 用于日期解析的时区。省略时使用 Worker JVM 默认时区,与 HugeGraph Source 一致,从而保证 Source→Sink 往返时绝对时间不变。 |
+
+### Legacy Schema配置 (`schema_config`)
+
+`schema_config` 定义一个输入流如何映射到 HugeGraph 中的某个顶点标签或边标签。该配置已废弃,新任务应使用 `mappings`。
| 名称 | 类型 | 是否必须 | 默认值 | 描述 |
| ------------------ | ------------------- | -------- | ------- |------------------------------------------------------------|
@@ -79,7 +123,14 @@ HugeGraph sink连接器允许您将数据从SeaTunnel写入Apache HugeGraph,
| 名称 | 类型 | 是否必须 | 默认值 | 描述 |
| ---------- | ------------ | -------- | ------ | -------------------------------------------------------------------------------------------------------------------------------------------- |
| `label` | String | 是 | - | 源或目标顶点的标签。 |
-| `idFields` | `List<String>` | 是 | - | 用于构造源/目标顶点ID的输入行中的源字段名称列表。这些值将被连接起来形成顶点ID。 |
+| `idFields` | `List<String>` | 是 | - | 用于构造源/目标顶点ID的输入行中的源字段名称列表。这些值将被连接起来形成顶点ID。对于 HugeGraph → HugeGraph 克隆,可将其设为已携带完整端点 id 的保留列——`sourceConfig` 用 `["~source_id"]`,`targetConfig` 用 `["~target_id"]`——连接器会直接复用该 id(见下方 *从 HugeGraph Source 克隆*)。 |
+
+### 从 HugeGraph Source 克隆(保留列 id 直连)
+
+当输入来自 HugeGraph Source 时,每行都带有保留列,携带已拼好的元素 id(顶点为 `~id`;边的端点为 `~source_id`/`~target_id`)。全保真克隆方式:
+
+- **顶点**(`CUSTOMIZE_STRING`/`CUSTOMIZE_NUMBER`/`CUSTOMIZE_UUID` id):将 `idStrategy` 设为对应的 `CUSTOMIZE_*`,`idFields = ["~id"]`,原始 id 原样写入。`PRIMARY_KEY` 顶点改用其主键属性列(Source 已输出);`AUTOMATIC` id 无法保留(目标服务端会重新分配)。
+- **边**:设 `sourceConfig.idFields = ["~source_id"]`、`targetConfig.idFields = ["~target_id"]`。端点 id 被直接复用,因此无论端点顶点的 id 策略为何都能克隆。目标端点的顶点 label 必须已存在(连接器不会用保留 id 自动建顶点 label)。
### Mapping配置 (`mapping`)
@@ -88,12 +139,20 @@ HugeGraph sink连接器允许您将数据从SeaTunnel写入Apache HugeGraph,
| 名称 | 类型 | 是否必须 | 默认值 | 描述 |
| ----------------- | ------------------ | -------- | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `fieldMapping` | `Map<String, String>` | 否 | - | 一个映射,其中键是源字段名,值是HugeGraph中的目标属性名。如果未指定,则使用源字段名作为目标属性名。 |
-| `valueMapping` | `Map<Object, Object>` | 否 | - | 用于转换特定字段值的映射。键是源的原始值,值是要写入的新值。 |
-| `nullableKeys` | `List<String>` | 否 | - | 可以具有null值的属性键列表。 |
+| `valueMapping` | `Map<String, Map<Object, Object>>` | 否 | - | 按字段的值转换映射。外层键为源字段名,内层为 `原始值 -> 新值`。按字段隔离,一个列的替换规则不会影响其他列。 |
+| `ignored` | `List<String>` | 否 | - | 从属性中排除的源字段黑名单(仅隐式模式生效)。与 `properties`(充当 selected 白名单)互斥。 |
+| `updateStrategies` | `Map<String, String>` | 否 | - | 写入时按目标属性名指定的属性级合并策略:`OVERRIDE`、`APPEND`、`SUM`、`UNION`、`BIGGER`、`SMALLER` 等。设置后对已存在元素做合并而非覆盖。 |
+| `nullableKeys` | `List<String>` | 否 | - | 自动建 label 时允许为 null 的属性键白名单。设置后覆盖默认可空行为。与 `notNullableKeys` 互斥。 |
+| `notNullableKeys` | `List<String>` | 否 | - | 与默认可空行为配合的反向 opt-out 列表,在此列出必须为非空的属性。与 `nullableKeys` 互斥。 |
| `nullValues` | `List<String>` | 否 | - | 应被视为`null`的字符串值列表。任何包含这些值的字段都不会被写入。 |
| `dateFormat` | String | 否 | `yyyy-MM-dd` | 用于解析日期字符串的日期格式。 |
-| `timeZone` | String | 否 | `GMT+8` | 用于日期解析的时区。 |
-| `sortKeys` | `List<String>` | 对于边 | - | 用于对具有相同源和目标顶点的边进行排序的属性键列表。 |
+| `extraDateFormats`| `List<String>` | 否 | - | 解析日期字符串时,在 `dateFormat` 之后按顺序尝试的额外日期格式——用于多源汇入、日期格式不一致的场景。 |
+| `listFormat` | Object | 否 | - | 原始字符串如何解析为 SET/LIST 属性元素:`startSymbol`(默认 `[`)、`endSymbol`(默认 `]`)、`elemDelimiter`(默认 `,`)、`ignoredElems`。 |
+| `unfold` | Boolean | 否 | false | (顶点)把 list 型 CUSTOMIZE id 单元格展开为每个元素一个顶点。仅 INSERT/append。 |
+| `unfoldSource` | Boolean | 否 | false | (边)把 list 型源端点 id 单元格展开为多条边(CUSTOMIZE 端点)。仅 INSERT/append。 |
+| `unfoldTarget` | Boolean | 否 | false | (边)把 list 型目标端点 id 单元格展开为多条边(与源端笛卡尔积)。仅 INSERT/append。 |
+| `timeZone` | String | 否 | Worker JVM 默认 | 用于日期解析的时区。省略时使用 Worker JVM 默认时区。 |
+| `sortKeys` | `List<String>` | 对于边 | - | **输入行中的源字段名**(`fieldMapping` 应用之前),用于区分相同源点和目标点之间的多条边。示例:当 `fieldMapping = {event_time: created_at}` 时,应填 `[event_time]`,而不是 `[created_at]`。 |
## 支持的数据类型
@@ -121,14 +180,18 @@ HugeGraph sink连接器允许您将数据从SeaTunnel写入Apache HugeGraph,
## 写入行为说明
-- 写入顶点时,`idStrategy` 决定如何生成顶点 ID。`PRIMARY_KEY` 会按 HugeGraph 主键格式拼接所有 `idFields`,`CUSTOMIZE_STRING` 会用 `:` 拼接字段,`CUSTOMIZE_NUMBER` 需要一个数字字段,`CUSTOMIZE_UUID` 需要一个 UUID 字段。
+- 写入顶点时,`idStrategy` 决定如何生成顶点 ID。`PRIMARY_KEY` 会按 HugeGraph 主键格式拼接所有 `idFields`;`CUSTOMIZE_STRING` 在多字段时用 `:` 拼接(并对字段值中的 `:` 做反斜杠转义,避免不同字段组合产生相同 id;单字段时原样使用);`CUSTOMIZE_NUMBER` 需要一个整数值字段(`1.9` 之类的小数会被拒绝,不会被静默截断);`CUSTOMIZE_UUID` 需要一个 UUID 字段。
- 写入边时,连接器会从 HugeGraph 中已有的源顶点标签和目标顶点标签读取 ID 策略。`sourceConfig.idFields` 和 `targetConfig.idFields` 必须能还原对应顶点 ID。
- `INSERT` 会写入新的顶点或边,`UPDATE_AFTER` 会更新已有图元素,`DELETE` 会删除图元素。删除行只需要包含能生成图元素 ID 的字段。
-- `mapping.nullValues` 中列出的字符串会被当作空值处理,写入时会跳过这些属性。
+- `AUTOMATIC` 顶点 ID 仅支持 INSERT;UPDATE 和 DELETE 必须使用可还原 ID 的策略。
+- Sink 提供 at-least-once 语义。使用 `AUTOMATIC` ID 时,重试或 checkpoint 恢复后的 INSERT 重放可能产生重复顶点。
+- 边批次使用所配置的 `check_vertex`(默认 `false`)。默认情况下顶点与边可能乱序写入,所有批次完成后图达到最终一致状态;设为 `check_vertex=true` 则服务端会拒绝端点尚不存在的边。
+- `nullValues` 中列出的字符串会被当作空值处理,写入时会跳过这些属性。
+- 时区通过**每个 mapping** 的 `timeZone` 配置(sink 没有顶层 `time_zone` 选项,与 HugeGraph Source 不同,因为日期解析属于每个 mapping 的行为)。省略时默认使用 Worker JVM 时区,与 Source 一致,从而保证 Source→Sink 往返时绝对时间不变。
## 使用示例
-下面示例默认 HugeGraph 中已经提前创建好对应 Schema。
+下面示例使用默认的 `schema_save_mode = CREATE_SCHEMA_WHEN_NOT_EXIST`。如果设置 `schema_save_mode = ERROR_WHEN_SCHEMA_NOT_EXIST`,请在运行任务前先创建好对应 HugeGraph Schema。
### 1. 写入顶点
@@ -156,15 +219,15 @@ sink {
host = "localhost"
port = 8080
graph_name = "hugegraph"
- graph_space = "default"
- selected_fields = ["name", "age"]
- schema_config = {
- type = "VERTEX"
- label = "person"
- idStrategy = "PRIMARY_KEY"
- idFields = ["name"]
- properties = ["name", "age"]
- }
+ mappings = [
+ {
+ type = "VERTEX"
+ label = "person"
+ idStrategy = "PRIMARY_KEY"
+ idFields = ["name"]
+ properties = ["name", "age"]
+ }
+ ]
}
}
```
@@ -196,26 +259,25 @@ sink {
host = "localhost"
port = 8080
graph_name = "hugegraph"
- graph_space = "default"
- schema_config = {
- type = "EDGE"
- label = "knows"
- sourceConfig = {
- label = "person"
- idFields = ["person1_name"]
- }
- targetConfig = {
- label = "person"
- idFields = ["person2_name"]
- }
- properties = ["since"]
- mapping = {
+ mappings = [
+ {
+ type = "EDGE"
+ label = "knows"
+ sourceConfig = {
+ label = "person"
+ idFields = ["person1_name"]
+ }
+ targetConfig = {
+ label = "person"
+ idFields = ["person2_name"]
+ }
+ properties = ["since"]
fieldMapping = {
person1_name = "name"
person2_name = "name"
}
}
- }
+ ]
}
}
```
diff --git a/docs/zh/connectors/source/HugeGraph.md b/docs/zh/connectors/source/HugeGraph.md
new file mode 100644
index 0000000000..e030146d76
--- /dev/null
+++ b/docs/zh/connectors/source/HugeGraph.md
@@ -0,0 +1,190 @@
+import ChangeLog from '../changelog/connector-hugegraph.md';
+
+# HugeGraph Source Connector
+
+`Source: HugeGraph`
+
+## 描述
+
+HugeGraph Source Connector 通过 HugeGraph REST API 读取 Apache HugeGraph 图数据。
+
+对一个顶点标签或一个边标签执行有界扫描——或在单个作业中读取某一类型的**全部** label——并保存读取进度以便作业在故障后恢复。
+
+- 当 `parallelism = 1` 时,通过服务端 list API 分页读取该 label,按 HugeGraph page-marker 读取到服务端返回 `page = null` 为止;此模式支持服务端 `filter`(属性等值过滤)。
+- 当 `parallelism > 1` 时,通过 HugeGraph `traverser().vertexShards / edgeShards` API 将 keyspace 切分为多个 shard,由多个 Reader 并行扫描。由于 shard 扫描按 key-range 返回所有 label,connector 会在客户端按配置的 `label` 过滤。详见[并行读取](#并行读取)。
+- 省略 `label` 时,在单个作业中读取 `label_type`(默认 `VERTEX`)下的全部 label,每个 label 产出一张输出表。详见[读取全部 label](#读取全部-label)。
+
+## 主要特性
+
+- [x] [batch](../../introduction/concepts/connector-v2-features.md)
+- [x] [parallelism](../../introduction/concepts/connector-v2-features.md)
+- [ ] [cdc](../../introduction/concepts/connector-v2-features.md)
+
+## 参数
+
+| 名称 | 类型 | 是否必填 | 默认值 | 描述 |
+|--------------------|---------|----------|----------|------|
+| `host` | String | 是 | - | HugeGraph 服务地址。 |
+| `port` | Integer | 是 | - | HugeGraph 服务端口。 |
+| `protocol` | String | 否 | `http` | 服务协议,支持 `http`、`https`。HTTPS 使用 JVM trust store。 |
+| `graph_name` | String | 是 | - | HugeGraph 图名称。 |
+| `label` | String | 否 | - | 要读取的顶点标签或边标签。**省略时,connector 会在单个作业中读取 `label_type` 下的全部 label,每个 label 产出一张表**(详见[读取全部 label](#读取全部-label));该模式下不允许配置 `schema` 与 `filter`。 |
+| `schema` | Object | 否 | - | 通过 `schema.fields` 声明输出属性列。保留图字段由 connector 自动添加。**省略时,connector 会从服务端读取 `label` 定义并自动发现全部属性列(类型自动推断,列按名称排序)。** 详见[Schema 自动发现](#schema-自动发现)。 |
+| `label_type` | Enum | 否 | `VERTEX` | 标签类型,支持 `VERTEX`、`EDGE`。 |
+| `page_size` | Integer | 否 | `1000` | 每页读取记录数,取值范围为 `[100, 10000]`。 |
+| `split_size` | Long | 否 | `1048576` | `parallelism > 1` 时每个 key-range shard 的目标字节大小。值越大 shard 越少越大。必须不小于 `1048576`(1 MiB,HugeGraph 的最小分片大小)——更小的值会在启动时被拒绝,以避免 shard 爆炸。`parallelism = 1` 时忽略。需要支持 scan 的后端(RocksDB / HBase / Cassandra)。 |
+| `filter` | Map | 否 | - | 可选的服务端属性等值过滤条件,例如 `{ country = "US", active = "true" }`。仅返回所有条件均匹配的元素。每个 key 必须是 `label` 的属性(未知 key 会在启动时报错),且每个值会被转换为该属性的类型(例如 `"true"` → 布尔、`"7"` → 对应数值类型)以便与服务端匹配——无法转换的值会在启动时报错,而不是静默返回 0 行。省略时读取该 label 的全部元素。**不能与 `parallelism > 1` 同时使用**(shard 扫描无法把属性过滤下推到服务端),两者同时设置会在启动时报错。 |
+| `time_zone` | String | 否 | Worker JVM 默认时区 | 用于转换服务端以 epoch/Date 返回的 HugeGraph DATE 值的 ZoneId,例如 `UTC` 或 `Asia/Shanghai`。对服务端已序列化为字符串(wall-clock)的 DATE 不生效(此类值不携带时区、原样保留)。Worker JVM 时区可能不一致时应显式设置。 |
+| `graph_space` | String | 否 | `DEFAULT` | 图所属的图空间(graph space)。 |
+| `username` | String | 否 | - | HugeGraph 用户名。 |
+| `password` | String | 否 | - | HugeGraph 密码。 |
+| `max_retries` | Integer | 否 | `3` | 首次请求失败后的重试次数。设置为 `0` 可禁用重试。 |
+| `retry_backoff_ms` | Integer | 否 | `5000` | 重试的基础退避时间(毫秒),按尝试次数指数增长(`retry_backoff_ms * 2^(attempt-1)`),上限为 `retry_backoff_max_ms`。 |
+| `retry_backoff_max_ms` | Integer | 否 | `30000` | 指数退避的上限(毫秒)。 |
+
+## 输出 Schema
+
+顶点输出列:
+
+```text
+~id, ~label, <schema.fields columns...>
+```
+
+边输出列:
+
+```text
+~id, ~label, ~source_id, ~source_label, ~target_id, ~target_label, <schema.fields columns...>
+```
+
+`~` 前缀字段为 connector 自动添加的保留字段。HugeGraph 属性键不能以 `~` 开头,因此不会与用户属性冲突。
+
+## 类型映射
+
+`schema.fields` 中声明的类型必须与 HugeGraph PropertyKey 类型匹配。Connector 会在读取前校验。
+
+| HugeGraph 类型 | SeaTunnel 类型 |
+|----------------|----------------|
+| `TEXT` | `STRING` |
+| `BYTE` | `TINYINT` |
+| `INT` | `INT` |
+| `LONG` | `BIGINT` |
+| `FLOAT` | `FLOAT` |
+| `DOUBLE` | `DOUBLE` |
+| `BOOLEAN` | `BOOLEAN` |
+| `DATE` | `TIMESTAMP` |
+| `UUID` | `STRING` |
+| `OBJECT` | `STRING` |
+| `BLOB` | `BYTES` |
+
+### 多值(LIST / SET)属性
+
+cardinality 为 `LIST` 或 `SET` 的 HugeGraph 属性会被读为 SeaTunnel 的 `ARRAY`。在 `schema.fields` 中声明为 `array<T>`,其中 `T` 是元素的 SeaTunnel 类型(见上表)。例如名为 `tags` 的 `LIST<TEXT>` 属性声明为 `tags = "array<string>"`。
+
+注意:
+
+- `SET` 元素在服务端无固定顺序;需要顺序时请使用 `LIST`。
+- 若服务端某属性 cardinality 为 `LIST`/`SET` 却被声明为标量(或反之),作业会在启动时失败并提示正确的声明方式。
+- 不支持 `LIST`/`SET` 中嵌套 `BLOB` 元素。
+
+## 示例
+
+```hocon
+source {
+ HugeGraph {
+ host = "localhost"
+ port = 8080
+ graph_name = "hugegraph"
+ label = "person"
+ label_type = "VERTEX"
+ page_size = 1000
+ schema = {
+ fields = {
+ name = "string"
+ age = "int"
+ }
+ }
+ }
+}
+```
+
+## Schema 自动发现
+
+`schema` 为可选项。省略时,connector 会在作业构建阶段连接服务端,读取 `label` 的定义,为每个属性键生成一个输出列(类型见[类型映射](#类型映射)表,`LIST`/`SET` 映射为 `array<T>`),并按属性名排序。适合「整 label 全字段 dump」而又不想逐字段手写声明的场景。
+
+```hocon
+source {
+ HugeGraph {
+ host = "localhost"
+ port = 8080
+ graph_name = "hugegraph"
+ label = "person"
+ label_type = "VERTEX"
+ # 不写 schema:读取 "person" 的全部属性
+ }
+}
+```
+
+注意:
+
+- 该 label 必须已存在于服务端,否则作业在构建阶段失败。
+- 无任何属性键的 label 只会产生保留列(`~id`、`~label` 等)。
+- 当只想读取部分属性、固定列顺序或指定类型时,请显式声明 `schema.fields`。
+
+## 读取全部 label
+
+省略 `label` 即可在单个作业中读取 `label_type`(默认 `VERTEX`)下的**全部** label——适合整图迁移 / 备份,而无需为每个 label 各配一个 source。作业构建阶段 connector 会从服务端 schema 列出该类型的所有 label,为每个 label 产出一张输出表,各自按 [Schema 自动发现](#schema-自动发现)推断列。每行都会带上其 label 对应的 table id,因此下游多表 sink 可据此将行路由到对应表。
+
+```hocon
+source {
+ HugeGraph {
+ host = "localhost"
+ port = 8080
+ graph_name = "hugegraph"
+ label_type = "VERTEX"
+ # 不写 label:读取全部顶点 label,每个 label 一张表
+ }
+}
+```
+
+注意:
+
+- 一个作业读取顶点**或**边,不能混读:设置 `label_type = "EDGE"` 以读取全部边 label。
+- 不允许配置 `schema`(单一 schema 无法描述多个 label),列始终按 label 自动发现。
+- 不允许配置 `filter`(属性等值过滤要求该属性存在于每个 label)。
+- 每个 label 对应一个 `LABEL_LIST` split,分配给各 Reader(并行度上限为 label 数量)。此模式不使用单个 label 内部的 shard 级并行。
+- 若图中不存在该类型的任何 label,作业在构建阶段失败。
+
+## 并行读取
+
+对于大图,设置 `parallelism > 1` 可并行读取一个 label。Enumerator 请求 HugeGraph 将该 label 的 keyspace 切分为大小约为 `split_size` 字节的多个 shard,并以 round-robin 方式分配给各 Reader,使吞吐随并行度提升,而不再受单一分页游标限制。
+
+```hocon
+source {
+ HugeGraph {
+ host = "localhost"
+ port = 8080
+ graph_name = "hugegraph"
+ label = "person"
+ label_type = "VERTEX"
+ parallelism = 8
+ split_size = 1048576
+ schema = {
+ fields = {
+ name = "string"
+ age = "int"
+ }
+ }
+ }
+}
+```
+
+注意:
+
+- Shard 扫描需要支持 scan 的后端(RocksDB / HBase / Cassandra);`memory` 后端不支持 shard 切分,请在其上使用 `parallelism = 1`。
+- Shard 扫描会返回 key-range 内所有 label 的元素,connector 仅保留配置的 `label`。当目标 label 只占全图很小比例时,单并行度的 `filter` 读取可能搬运更少数据(尽管无法并行)。
+- `filter` 不能与 `parallelism > 1` 同时使用;要用服务端过滤请保持 `parallelism = 1`,要并行请去掉 filter。
+- 调优 `split_size`:值越小 shard 越多越小(负载更均衡、请求更多);值越大 shard 越少越大。最小值为 `1048576`(1 MiB),更小的值会被拒绝,以避免把 keyspace 切分成过多的 shard。
+
+## Changelog
+
+<ChangeLog />
diff --git a/plugin-mapping.properties b/plugin-mapping.properties
index af93a9c8f7..d4b0b1e138 100644
--- a/plugin-mapping.properties
+++ b/plugin-mapping.properties
@@ -159,6 +159,7 @@ seatunnel.source.GraphQL = connector-graphql
seatunnel.sink.GraphQL = connector-graphql
seatunnel.sink.Aerospike = connector-aerospike
seatunnel.sink.SensorsData = connector-sensorsdata
+seatunnel.source.HugeGraph = connector-hugegraph
seatunnel.sink.HugeGraph = connector-hugegraph
seatunnel.source.Fluss = connector-fluss
seatunnel.sink.Fluss = connector-fluss
diff --git a/pom.xml b/pom.xml
index 058d75d41a..8a49c5f06a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -148,7 +148,7 @@
<scala.version>2.12.15</scala.version>
<jetty.version>9.4.56.v20240826</jetty.version>
<jakarta.servlet-api>4.0.4</jakarta.servlet-api>
- <hugegraph.client.version>1.5.0</hugegraph.client.version>
+ <hugegraph.client.version>1.7.0</hugegraph.client.version>
<!-- Option args -->
<skipUT>false</skipUT>
<skipIT>true</skipIT>
diff --git a/seatunnel-connectors-v2/connector-hugegraph/pom.xml b/seatunnel-connectors-v2/connector-hugegraph/pom.xml
index d8cf4077fd..2faf3ae6c9 100644
--- a/seatunnel-connectors-v2/connector-hugegraph/pom.xml
+++ b/seatunnel-connectors-v2/connector-hugegraph/pom.xml
@@ -29,10 +29,6 @@
<artifactId>connector-hugegraph</artifactId>
<name>SeaTunnel : Connectors V2 : HugeGraph</name>
- <properties>
- <hugegraph.client.version>1.5.0</hugegraph.client.version>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.apache.seatunnel</groupId>
@@ -44,12 +40,53 @@
<groupId>org.apache.hugegraph</groupId>
<artifactId>hugegraph-client</artifactId>
<version>${hugegraph.client.version}</version>
+ <exclusions>
+ <!-- hugegraph-common leaks junit/hamcrest as compile-scope transitives;
+ they must not be shaded into the connector runtime jar. -->
+ <exclusion>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-core</artifactId>
+ </exclusion>
+ <!-- The connector only uses the HTTP REST client (HugeClient.builder), not the
+ distributed PD/HStore path, so drop the heavy PD + gRPC transitives that the
+ 1.7.0 client pulls in (~18MB of shaded jar). -->
+ <exclusion>
+ <groupId>org.apache.hugegraph</groupId>
+ <artifactId>hg-pd-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hugegraph</groupId>
+ <artifactId>hg-pd-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hugegraph</groupId>
+ <artifactId>hg-pd-grpc</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.grpc</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hugegraph</groupId>
<artifactId>hugegraph-common</artifactId>
<version>${hugegraph.client.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
</dependencies>
@@ -67,6 +104,68 @@
</archive>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <!-- Relocate dependencies that clash with engine-provided jars.
+ Spark ships an old Guava (14.x) and Jackson (2.13.x) on its
+ classpath; without relocation the engine versions win and
+ hugegraph-client fails with NoSuchMethodError. -->
+ <relocations>
+ <relocation>
+ <pattern>com.google.common</pattern>
+ <shadedPattern>${seatunnel.shade.package}.hugegraph.com.google.common</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.google.thirdparty</pattern>
+ <shadedPattern>${seatunnel.shade.package}.hugegraph.com.google.thirdparty</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.fasterxml.jackson</pattern>
+ <shadedPattern>${seatunnel.shade.package}.hugegraph.com.fasterxml.jackson</shadedPattern>
+ </relocation>
+ <!-- Spark bundles okhttp 3.12 / okio 1.x (kubernetes-client),
+ hugegraph-client needs okhttp 4.12 / okio 3.x -->
+ <relocation>
+ <pattern>okhttp3</pattern>
+ <shadedPattern>${seatunnel.shade.package}.hugegraph.okhttp3</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>okio</pattern>
+ <shadedPattern>${seatunnel.shade.package}.hugegraph.okio</shadedPattern>
+ </relocation>
+ <!-- Spark bundles javassist 3.25 (hive) and joda-time 2.9 (Spark 2.4) -->
+ <relocation>
+ <pattern>javassist</pattern>
+ <shadedPattern>${seatunnel.shade.package}.hugegraph.javassist</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.joda.time</pattern>
+ <shadedPattern>${seatunnel.shade.package}.hugegraph.org.joda.time</shadedPattern>
+ </relocation>
+ </relocations>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ </transformers>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/maven/**</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/buffer/BatchBuffer.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/buffer/BatchBuffer.java
index c385112939..bd6575488e 100644
--- a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/buffer/BatchBuffer.java
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/buffer/BatchBuffer.java
@@ -18,30 +18,50 @@
package org.apache.seatunnel.connectors.seatunnel.hugegraph.buffer;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.client.HugeGraphClient;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.MappingConfig.LabelType;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorErrorCode;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorException;
import org.apache.hugegraph.structure.GraphElement;
import org.apache.hugegraph.structure.graph.Edge;
+import org.apache.hugegraph.structure.graph.UpdateStrategy;
import org.apache.hugegraph.structure.graph.Vertex;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
+/**
+ * Dual-bucket batch buffer that independently accumulates and flushes vertices and edges. Each
+ * bucket triggers flush when reaching batch_size; both buckets are flushed on timer, prepareCommit,
+ * or close.
+ *
+ * <p>Vertex-before-edge ordering is enforced only when {@code check_vertex} is true — the server
+ * then rejects edges whose endpoint vertices do not yet exist, so a filling edge bucket first
+ * flushes any pending vertices, and {@link #flush()} writes vertices before edges. When {@code
+ * check_vertex} is false (the default) the server already accepts orphan edges, so the buckets
+ * flush independently for higher throughput (no forced, undersized vertex flushes).
+ */
public class BatchBuffer implements AutoCloseable {
private static final Logger LOG = LoggerFactory.getLogger(BatchBuffer.class);
- private final List<GraphElement> buffer = new ArrayList<>();
+ private final List<GraphElementEnvelope> vertexBuffer = new ArrayList<>();
+ private final List<GraphElementEnvelope> edgeBuffer = new ArrayList<>();
private final int batchSize;
private final ScheduledExecutorService scheduler;
private final ScheduledFuture<?> scheduledFuture;
@@ -49,11 +69,61 @@ public class BatchBuffer implements AutoCloseable {
private volatile boolean closed = false;
private volatile Exception flushException;
private final HugeGraphClient client;
+ private final boolean batchFailureFallback;
+ private final boolean checkVertex;
+ // Fail the task once this many records have been skipped by the single-record fallback;
+ // negative means unlimited. Guards against the previously unbounded silent skipping.
+ private final int maxInsertErrors;
+ // Optional directory for skipped-record failure samples; null = do not persist.
+ private final String failureDataPath;
+ private final int subtaskIndex;
+ // Cumulative count of records skipped by the fallback across this writer's lifetime. Only
+ // mutated inside synchronized flush paths, so a plain long is sufficient.
+ private long insertFailureCount;
+ // Lazily opened on the first persisted sample; disabled after an I/O error so a broken
+ // failure-log path never turns into a second failure that masks the real one.
+ private BufferedWriter failureWriter;
+ private boolean failureWriterDisabled;
+ /**
+ * Backward-compatible constructor that retains the original 3-argument signature. Defaults
+ * {@code batchFailureFallback} and {@code checkVertex} to {@code false}, matching the pre-2.x
+ * behaviour where neither feature existed.
+ *
+ * @deprecated Use {@link #BatchBuffer(HugeGraphClient, int, long, boolean, boolean)} instead so
+ * callers explicitly opt into failure-fallback and vertex-checking semantics.
+ */
+ @Deprecated
public BatchBuffer(HugeGraphClient client, int batchSize, long batchIntervalMs) {
+ this(client, batchSize, batchIntervalMs, false, false);
+ }
+
+ public BatchBuffer(
+ HugeGraphClient client,
+ int batchSize,
+ long batchIntervalMs,
+ boolean batchFailureFallback,
+ boolean checkVertex) {
+ this(client, batchSize, batchIntervalMs, batchFailureFallback, checkVertex, -1, null, 0);
+ }
+ public BatchBuffer(
+ HugeGraphClient client,
+ int batchSize,
+ long batchIntervalMs,
+ boolean batchFailureFallback,
+ boolean checkVertex,
+ int maxInsertErrors,
+ String failureDataPath,
+ int subtaskIndex) {
this.batchSize = batchSize;
this.client = client;
+ this.batchFailureFallback = batchFailureFallback;
+ this.checkVertex = checkVertex;
+ this.maxInsertErrors = maxInsertErrors;
+ this.failureDataPath = failureDataPath;
+ this.subtaskIndex = subtaskIndex;
+ this.insertFailureCount = 0;
if (batchIntervalMs > 0) {
this.scheduler =
@@ -81,7 +151,7 @@ public class BatchBuffer implements AutoCloseable {
}
}
- public synchronized void add(GraphElement element) throws IOException {
+ public synchronized void add(GraphElementEnvelope envelope) throws IOException {
checkFlushException();
if (closed) {
throw new HugeGraphConnectorException(
@@ -90,9 +160,25 @@ public class BatchBuffer implements AutoCloseable {
}
try {
- buffer.add(element);
- if (buffer.size() >= batchSize) {
- doFlush();
+ if (envelope.getElementType() == LabelType.VERTEX) {
+ vertexBuffer.add(envelope);
+ if (vertexBuffer.size() >= batchSize) {
+ doFlushVertices();
+ }
+ } else {
+ edgeBuffer.add(envelope);
+ if (edgeBuffer.size() >= batchSize) {
+ // Topology safety only matters when the server validates endpoints: with
+ // check_vertex=true, flush pending vertices before the edges so no edge is sent
+ // before its endpoints exist. With check_vertex=false the server already
+ // accepts
+ // orphan edges, so skip the forced (undersized) vertex flush and let the vertex
+ // bucket accumulate to a full batch — fewer, fuller vertex requests.
+ if (checkVertex && !vertexBuffer.isEmpty()) {
+ doFlushVertices();
+ }
+ doFlushEdges();
+ }
}
} catch (Exception e) {
throw new HugeGraphConnectorException(
@@ -100,40 +186,267 @@ public class BatchBuffer implements AutoCloseable {
}
}
+ /**
+ * Backward-compatible overload that wraps a plain {@link GraphElement} in a minimal envelope.
+ *
+ * @deprecated Use {@link #add(GraphElementEnvelope)} instead so the buffer receives complete
+ * mapping context (label name, element type) for failure diagnostics.
+ */
+ @Deprecated
+ public synchronized void add(GraphElement element) throws IOException {
+ LabelType type = element instanceof Vertex ? LabelType.VERTEX : LabelType.EDGE;
+ add(new GraphElementEnvelope(null, type, element));
+ }
+
public synchronized void flush() throws IOException {
checkFlushException();
- if (closed && buffer.isEmpty()) {
+ if (closed && vertexBuffer.isEmpty() && edgeBuffer.isEmpty()) {
return;
}
- doFlush();
+ doFlushVertices();
+ doFlushEdges();
}
- private void doFlush() {
- if (buffer.isEmpty()) {
+ private void doFlushVertices() {
+ if (vertexBuffer.isEmpty()) {
return;
}
+ List<GraphElementEnvelope> batch = new ArrayList<>(vertexBuffer);
+ vertexBuffer.clear();
+ // Route by each element's own mapping strategy: a group with no strategy is a plain insert,
+ // a group with a strategy is an upsert. HugeGraph applies one strategy map per batch call,
+ // so elements with different strategies must go in separate calls.
+ for (Map.Entry<Map<String, UpdateStrategy>, List<GraphElementEnvelope>> group :
+ groupByStrategy(batch).entrySet()) {
+ flushVertexGroup(group.getValue(), group.getKey());
+ }
+ }
+
+ private void flushVertexGroup(
+ List<GraphElementEnvelope> batch, Map<String, UpdateStrategy> updateStrategies) {
try {
- GraphElement firstElement = buffer.get(0);
- if (firstElement instanceof Vertex) {
- List<Vertex> vertices =
- buffer.stream()
- .map(element -> (Vertex) element)
- .collect(Collectors.toList());
+ List<Vertex> vertices =
+ batch.stream()
+ .map(env -> (Vertex) env.getElement())
+ .collect(Collectors.toList());
+ if (updateStrategies.isEmpty()) {
client.batchWriteVertices(vertices);
} else {
- List<Edge> edges =
- buffer.stream().map(element -> (Edge) element).collect(Collectors.toList());
- client.batchWriteEdges(edges);
+ client.batchUpdateVertices(vertices, updateStrategies);
+ }
+ } catch (Exception e) {
+ if (!batchFailureFallback) {
+ logBatchFailure(batch, e);
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
+ "Failed to write vertex batch",
+ e);
}
+ fallbackInsertSingly(batch, e);
+ }
+ }
+
+ private void doFlushEdges() {
+ if (edgeBuffer.isEmpty()) {
+ return;
+ }
+ List<GraphElementEnvelope> batch = new ArrayList<>(edgeBuffer);
+ edgeBuffer.clear();
+ for (Map.Entry<Map<String, UpdateStrategy>, List<GraphElementEnvelope>> group :
+ groupByStrategy(batch).entrySet()) {
+ flushEdgeGroup(group.getValue(), group.getKey());
+ }
+ }
- buffer.clear();
+ private void flushEdgeGroup(
+ List<GraphElementEnvelope> batch, Map<String, UpdateStrategy> updateStrategies) {
+ try {
+ List<Edge> edges =
+ batch.stream().map(env -> (Edge) env.getElement()).collect(Collectors.toList());
+ if (updateStrategies.isEmpty()) {
+ client.batchWriteEdges(edges, checkVertex);
+ } else {
+ client.batchUpdateEdges(edges, updateStrategies, checkVertex);
+ }
} catch (Exception e) {
- LOG.error("Failed to write batch data to HugeGraph", e);
+ if (!batchFailureFallback) {
+ logBatchFailure(batch, e);
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
+ "Failed to write edge batch",
+ e);
+ }
+ fallbackInsertSingly(batch, e);
+ }
+ }
+
+ /**
+ * Groups a batch by its elements' update-strategy map, preserving first-seen order so a flush
+ * stays deterministic. Elements sharing the same strategy map flush together in one server
+ * call.
+ */
+ private static Map<Map<String, UpdateStrategy>, List<GraphElementEnvelope>> groupByStrategy(
+ List<GraphElementEnvelope> batch) {
+ Map<Map<String, UpdateStrategy>, List<GraphElementEnvelope>> groups =
+ new java.util.LinkedHashMap<>();
+ for (GraphElementEnvelope envelope : batch) {
+ groups.computeIfAbsent(envelope.getUpdateStrategies(), key -> new ArrayList<>())
+ .add(envelope);
+ }
+ return groups;
+ }
+
+ /**
+ * A batch insert failed; retry each element on its own so a single poison record no longer
+ * fails the whole batch. Failed records are logged and skipped; the rest succeed. If
+ * <em>every</em> record fails, the failure is systemic (bad connection / schema), not a poison
+ * record, so it is rethrown instead of silently dropping the whole batch.
+ */
+ private void fallbackInsertSingly(List<GraphElementEnvelope> batch, Exception batchFailure) {
+ LOG.warn(
+ "Batch write failed ({} element(s)); falling back to single-record insert. cause={}",
+ batch.size(),
+ batchFailure.getMessage());
+ int failed = 0;
+ Exception lastFailure = null;
+ for (GraphElementEnvelope envelope : batch) {
+ Map<String, UpdateStrategy> updateStrategies = envelope.getUpdateStrategies();
+ try {
+ if (envelope.getElementType() == LabelType.VERTEX) {
+ if (updateStrategies.isEmpty()) {
+ client.writeVertex((Vertex) envelope.getElement());
+ } else {
+ client.updateVertex((Vertex) envelope.getElement(), updateStrategies);
+ }
+ } else {
+ if (updateStrategies.isEmpty()) {
+ client.writeEdge((Edge) envelope.getElement(), checkVertex);
+ } else {
+ client.updateEdge(
+ (Edge) envelope.getElement(), updateStrategies, checkVertex);
+ }
+ }
+ } catch (Exception single) {
+ failed++;
+ lastFailure = single;
+ insertFailureCount++;
+ LOG.error(
+ "Single-record write failure — {}",
+ formatFailureDiagnostic(envelope, single));
+ writeFailureSample(envelope, single);
+ // Bound the previously unlimited silent skipping: once the cumulative number of
+ // skipped records reaches max_insert_errors, stop and fail the task instead of
+ // continuing to drop data. Negative means unlimited.
+ if (maxInsertErrors >= 0 && insertFailureCount >= maxInsertErrors) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
+ String.format(
+ "Aborting: cumulative single-insert failures (%d) reached "
+ + "max_insert_errors (%d). Last error: %s",
+ insertFailureCount, maxInsertErrors, single.getMessage()),
+ single);
+ }
+ }
+ }
+ if (failed == batch.size()) {
throw new HugeGraphConnectorException(
- HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED, e.getMessage(), e);
+ HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
+ String.format(
+ "All %d record(s) in the batch failed single-insert fallback",
+ batch.size()),
+ lastFailure);
+ }
+ if (failed > 0) {
+ LOG.warn(
+ "Single-record fallback completed: {} succeeded, {} failed and were skipped "
+ + "({} skipped in total so far)",
+ batch.size() - failed,
+ failed,
+ insertFailureCount);
+ }
+ }
+
+ /**
+ * Appends one line describing a skipped record — the mapped element's id/label/properties plus
+ * the server error — to the per-subtask failure file when {@code failure_data_path} is set.
+ * Best-effort: a write/open error disables further persistence rather than failing the task, so
+ * a broken debug path can never mask the real insert failure.
+ */
+ private void writeFailureSample(GraphElementEnvelope envelope, Exception failure) {
+ if (failureDataPath == null || failureDataPath.isEmpty() || failureWriterDisabled) {
+ return;
+ }
+ try {
+ if (failureWriter == null) {
+ File dir = new File(failureDataPath);
+ if (!dir.exists() && !dir.mkdirs() && !dir.exists()) {
+ throw new IOException("Failed to create failure data directory: " + dir);
+ }
+ File file =
+ new File(dir, "hugegraph-sink-failures-subtask-" + subtaskIndex + ".log");
+ failureWriter =
+ new BufferedWriter(
+ new OutputStreamWriter(
+ new FileOutputStream(file, true), StandardCharsets.UTF_8));
+ LOG.info("Persisting skipped-record failure samples to {}", file.getAbsolutePath());
+ }
+ failureWriter.write(formatFailureSample(envelope, failure));
+ failureWriter.newLine();
+ // Flush per record: failures are rare and losing samples on an abrupt crash defeats
+ // their purpose.
+ failureWriter.flush();
+ } catch (IOException e) {
+ failureWriterDisabled = true;
+ LOG.warn(
+ "Failed to persist failure sample to '{}'; disabling failure-data persistence. cause={}",
+ failureDataPath,
+ e.getMessage());
}
}
+ /**
+ * One-line, tab-delimited failure sample. Newlines are stripped to keep one record per line.
+ */
+ static String formatFailureSample(GraphElementEnvelope envelope, Exception failure) {
+ GraphElement element = envelope.getElement();
+ String line =
+ String.format(
+ "mapping=%s\ttype=%s\tid=%s\tlabel=%s\tproperties=%s\terror=%s",
+ envelope.getMappingLabel(),
+ envelope.getElementType(),
+ element == null ? null : element.id(),
+ element == null ? null : element.label(),
+ element == null ? null : element.properties(),
+ failure.getMessage());
+ return line.replace('\n', ' ').replace('\r', ' ');
+ }
+
+ private void logBatchFailure(List<GraphElementEnvelope> batch, Exception e) {
+ LOG.error(
+ "Batch write failure — {} element(s), failureType={}, serverError={}",
+ batch.size(),
+ e.getClass().getName(),
+ e.getMessage());
+ for (GraphElementEnvelope envelope : batch) {
+ LOG.error("Graph element write failure — {}", formatFailureDiagnostic(envelope, e));
+ }
+ }
+
+ static String formatFailureDiagnostic(GraphElementEnvelope envelope, Exception failure) {
+ // Log only the mapped graph element's id/label — bounded and non-sensitive. The raw source
+ // row is intentionally not retained (see GraphElementEnvelope) to avoid unbounded memory
+ // and
+ // leaking excluded field content into logs.
+ return String.format(
+ "mapping=%s, elementType=%s, elementId=%s, elementLabel=%s, failureType=%s, serverError=%s",
+ envelope.getMappingLabel(),
+ envelope.getElementType(),
+ envelope.getElement() == null ? null : envelope.getElement().id(),
+ envelope.getElement() == null ? null : envelope.getElement().label(),
+ failure.getClass().getName(),
+ failure.getMessage());
+ }
+
@Override
public void close() throws IOException {
synchronized (this) {
@@ -158,11 +471,27 @@ public class BatchBuffer implements AutoCloseable {
}
}
LOG.info("Closing BatchBuffer, performing final flush...");
- flush();
- checkFlushException();
+ try {
+ flush();
+ checkFlushException();
+ } finally {
+ closeFailureWriter();
+ }
LOG.info("BatchBuffer closed.");
}
+ private void closeFailureWriter() {
+ if (failureWriter != null) {
+ try {
+ failureWriter.close();
+ } catch (IOException e) {
+ LOG.warn("Failed to close failure-data writer. cause={}", e.getMessage());
+ } finally {
+ failureWriter = null;
+ }
+ }
+ }
+
private void checkFlushException() {
if (flushException != null) {
throw new HugeGraphConnectorException(
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/buffer/GraphElementEnvelope.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/buffer/GraphElementEnvelope.java
new file mode 100644
index 0000000000..3b1bc2ad0e
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/buffer/GraphElementEnvelope.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.hugegraph.buffer;
+
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.MappingConfig.LabelType;
+
+import org.apache.hugegraph.structure.GraphElement;
+import org.apache.hugegraph.structure.graph.UpdateStrategy;
+
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Wraps a graph element with non-sensitive mapping context for failure diagnostics.
+ *
+ * <p>Deliberately does NOT retain the source {@link
+ * org.apache.seatunnel.api.table.type.SeaTunnelRow}: only the mapped {@link GraphElement} is ever
+ * sent, and envelopes stay alive until the batch is flushed (by size/timer/checkpoint/close).
+ * Keeping the raw row would pin fields that were excluded by {@code mapping.properties} (e.g. large
+ * BYTES payloads) in memory for the whole batch and leak their content into failure logs.
+ */
+public class GraphElementEnvelope {
+
+ private final String mappingLabel;
+ private final LabelType elementType;
+ private final GraphElement element;
+ // Per-mapping update strategies (property name -> strategy). Empty means plain insert. Carried
+ // on the envelope so the buffer can route each element by its own mapping's strategy instead of
+ // one merged global map — a strategy on one mapping no longer forces upsert on every mapping,
+ // and two mappings may assign different strategies to the same property name.
+ private final Map<String, UpdateStrategy> updateStrategies;
+
+ public GraphElementEnvelope(String mappingLabel, LabelType elementType, GraphElement element) {
+ this(mappingLabel, elementType, element, Collections.emptyMap());
+ }
+
+ public GraphElementEnvelope(
+ String mappingLabel,
+ LabelType elementType,
+ GraphElement element,
+ Map<String, UpdateStrategy> updateStrategies) {
+ this.mappingLabel = mappingLabel;
+ this.elementType = elementType;
+ this.element = element;
+ this.updateStrategies =
+ updateStrategies == null ? Collections.emptyMap() : updateStrategies;
+ }
+
+ public String getMappingLabel() {
+ return mappingLabel;
+ }
+
+ public LabelType getElementType() {
+ return elementType;
+ }
+
+ public GraphElement getElement() {
+ return element;
+ }
+
+ public Map<String, UpdateStrategy> getUpdateStrategies() {
+ return updateStrategies;
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/client/HugeGraphClient.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/client/HugeGraphClient.java
index ed4491eaef..d173e9bca0 100644
--- a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/client/HugeGraphClient.java
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/client/HugeGraphClient.java
@@ -6,7 +6,7 @@
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,18 +17,31 @@
package org.apache.seatunnel.connectors.seatunnel.hugegraph.client;
-import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.HugeGraphSinkConfig;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.HugeGraphConnectionConfig;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.LabelOptions;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorErrorCode;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorException;
+import org.apache.hugegraph.api.graph.EdgeAPI;
+import org.apache.hugegraph.api.graph.VertexAPI;
+import org.apache.hugegraph.client.RestClient;
import org.apache.hugegraph.driver.GraphManager;
import org.apache.hugegraph.driver.HugeClient;
import org.apache.hugegraph.driver.SchemaManager;
import org.apache.hugegraph.exception.ServerException;
import org.apache.hugegraph.rest.ClientException;
+import org.apache.hugegraph.rest.RestClientConfig;
+import org.apache.hugegraph.structure.constant.DataType;
+import org.apache.hugegraph.structure.constant.Frequency;
import org.apache.hugegraph.structure.constant.IdStrategy;
+import org.apache.hugegraph.structure.graph.BatchEdgeRequest;
+import org.apache.hugegraph.structure.graph.BatchVertexRequest;
import org.apache.hugegraph.structure.graph.Edge;
+import org.apache.hugegraph.structure.graph.Edges;
+import org.apache.hugegraph.structure.graph.Shard;
+import org.apache.hugegraph.structure.graph.UpdateStrategy;
import org.apache.hugegraph.structure.graph.Vertex;
+import org.apache.hugegraph.structure.graph.Vertices;
import org.apache.hugegraph.structure.schema.EdgeLabel;
import org.apache.hugegraph.structure.schema.PropertyKey;
import org.apache.hugegraph.structure.schema.VertexLabel;
@@ -36,39 +49,61 @@ import org.apache.hugegraph.structure.schema.VertexLabel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
-public final class HugeGraphClient {
+public final class HugeGraphClient implements HugeGraphOperations {
- // TODO: Add handling for schema fetch failures.
private static final Logger LOG = LoggerFactory.getLogger(HugeGraphClient.class);
+ /** HugeGraph server per-request batch cap (server option batch.max_vertices_per_batch). */
+ private static final int MAX_RECORDS_PER_BATCH_REQUEST = 500;
+
private HugeClient client;
+ private RestClient restClient;
+ private VertexAPI vertexAPI;
+ private EdgeAPI edgeAPI;
private SchemaManager schema;
- private final HugeGraphSinkConfig config;
+ private final HugeGraphConnectionConfig config;
private final int maxRetries;
private final long retryBackoffMs;
+ private final long retryBackoffMaxMs;
- public HugeGraphClient(HugeGraphSinkConfig config) {
+ public HugeGraphClient(HugeGraphConnectionConfig config) {
this.client = null;
+ this.restClient = null;
+ this.vertexAPI = null;
+ this.edgeAPI = null;
this.schema = null;
this.config = config;
- this.maxRetries = config.getMaxRetries() > 0 ? config.getMaxRetries() : 3;
- this.retryBackoffMs = config.getRetryBackoffMs() > 0 ? config.getRetryBackoffMs() : 5000L;
+ this.maxRetries = Math.max(0, config.getMaxRetries());
+ this.retryBackoffMs = Math.max(0, config.getRetryBackoffMs());
+ this.retryBackoffMaxMs = Math.max(0, config.getRetryBackoffMaxMs());
}
- private HugeClient createClient(HugeGraphSinkConfig config) {
+ /** Default graph space per HugeGraphOptions.GRAPH_SPACE.defaultValue(). */
+ private static final String DEFAULT_GRAPH_SPACE = "DEFAULT";
+
+ private HugeClient createClient(HugeGraphConnectionConfig config) {
try {
- String url = String.format("http://%s:%d", config.getHost(), config.getPort());
- LOG.debug("Creating new HugeClient for url: {}, graph: {}", url, config.getGraphName());
+ String url = buildServerUrl(config);
+ String graphSpace =
+ config.getGraphSpace() != null ? config.getGraphSpace() : DEFAULT_GRAPH_SPACE;
+ LOG.debug(
+ "Creating new HugeClient for url: {}, graphSpace: {}, graph: {}",
+ url,
+ graphSpace,
+ config.getGraphName());
HugeClient client =
- HugeClient.builder(url, config.getGraphName())
+ HugeClient.builder(url, graphSpace, config.getGraphName())
.configUser(config.getUsername(), config.getPassword())
.configIdleTime(60)
.build();
- client.graph().listVertices();
LOG.info("Successfully created and validated HugeClient instance.");
return client;
} catch (Exception e) {
@@ -83,14 +118,23 @@ public final class HugeGraphClient {
void execute(GraphManager graph) throws ServerException, ClientException;
}
+ @FunctionalInterface
+ private interface ReadOperation<T> {
+ T execute() throws ServerException, ClientException;
+ }
+
private void ensureClientInitialized() throws HugeGraphConnectorException {
if (this.client == null) {
LOG.info("Client not initialized. Attempting to connect...");
try {
this.client = createClient(this.config);
this.schema = this.client.schema();
+ createPageApis(this.config);
LOG.info("HugeClient initialized successfully.");
} catch (Exception e) {
+ // Avoid leaking a partially-opened client (e.g. createPageApis failed after the
+ // HugeClient was created) — release everything before surfacing the failure.
+ reconnect();
throw new HugeGraphConnectorException(
HugeGraphConnectorErrorCode.BUILD_CLIENT_FAILED,
"Failed to establish initial connection",
@@ -109,52 +153,263 @@ public final class HugeGraphClient {
}
}
this.client = null;
+ if (this.restClient != null) {
+ try {
+ this.restClient.close();
+ } catch (Exception e) {
+ LOG.warn("Error closing potentially broken REST client: {}", e.getMessage());
+ }
+ }
+ this.restClient = null;
+ this.vertexAPI = null;
+ this.edgeAPI = null;
this.schema = null;
}
- private void executeGraphOperation(GraphOperation operation) {
- for (int attempt = 1; attempt <= this.maxRetries; attempt++) {
+ private void createPageApis(HugeGraphConnectionConfig config) {
+ String url = buildServerUrl(config);
+ String graphSpace =
+ config.getGraphSpace() != null ? config.getGraphSpace() : DEFAULT_GRAPH_SPACE;
+ RestClientConfig restClientConfig =
+ RestClientConfig.builder()
+ .user(config.getUsername() == null ? "" : config.getUsername())
+ .password(config.getPassword() == null ? "" : config.getPassword())
+ .build();
+ this.restClient = new RestClient(url, restClientConfig);
+ this.vertexAPI = new VertexAPI(this.restClient, graphSpace, config.getGraphName());
+ this.edgeAPI = new EdgeAPI(this.restClient, graphSpace, config.getGraphName());
+ }
+
+ static String buildServerUrl(HugeGraphConnectionConfig config) {
+ String protocol =
+ config.getProtocol() == null || config.getProtocol().isEmpty()
+ ? "http"
+ : config.getProtocol().toLowerCase(java.util.Locale.ROOT);
+ return String.format("%s://%s:%d", protocol, config.getHost(), config.getPort());
+ }
+
+ /**
+ * Executes a write operation that is safe to retry: UPSERT (updateVertices/updateEdges with
+ * createIfNotExist=true) and DELETE (removeVertex/removeEdge). Idempotent operations are
+ * retried on retryable errors because a second attempt cannot create duplicates.
+ */
+ private void executeIdempotentWrite(GraphOperation operation) {
+ executeGraphOperation(operation, true);
+ }
+
+ /**
+ * Executes a write operation that is NOT safe to retry: plain INSERT (addVertex/addVertices/
+ * addEdge/addEdges). A retry after a server-committed-but-client-timed-out response would
+ * create a duplicate element. Non-idempotent writes fail fast — the caller's single-record
+ * fallback handles them individually instead.
+ */
+ private void executeNonIdempotentWrite(GraphOperation operation) {
+ try {
+ ensureClientInitialized();
+ operation.execute(this.client.graph());
+ } catch (ServerException | ClientException e) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
+ "Non-idempotent write failed (not retried to avoid duplicates): "
+ + e.getMessage(),
+ e);
+ }
+ }
+
+ /**
+ * Executes a graph write with optional retry. When {@code idempotent} is true, retryable server
+ * errors (status ≥ 500, 408, 425, 429) are retried up to {@code maxRetries} times with
+ * exponential backoff. When false, the operation is attempted once — if it fails the exception
+ * propagates immediately so the caller can route through the single-record fallback or skip the
+ * record.
+ */
+ private void executeGraphOperation(GraphOperation operation, boolean idempotent) {
+ int totalAttempts = idempotent ? this.maxRetries + 1 : 1;
+ for (int attempt = 1; attempt <= totalAttempts; attempt++) {
try {
ensureClientInitialized();
operation.execute(this.client.graph());
return;
} catch (ServerException | ClientException e) {
+ if (!isRetryable(e) || !idempotent) {
+ LOG.error(
+ "Server rejected the request ({}): {}",
+ idempotent ? "non-retryable" : "non-idempotent, not retrying",
+ e.getMessage());
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
+ "Server rejected the request"
+ + (idempotent ? " (non-retryable)" : " (non-idempotent)")
+ + ": "
+ + e.getMessage(),
+ e);
+ }
LOG.warn(
"Graph operation failed on attempt {}/{}. Error: {}",
attempt,
- this.maxRetries,
+ totalAttempts,
e.getMessage());
reconnect();
- if (attempt == this.maxRetries) {
+ if (attempt == totalAttempts) {
LOG.error("Max retries ({}) reached. Failing task.", this.maxRetries);
throw new HugeGraphConnectorException(
HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
"Failed to execute graph operation after "
- + this.maxRetries
- + " attempts",
+ + totalAttempts
+ + " attempt(s). Last error: "
+ + e.getMessage(),
e);
}
- try {
- LOG.info("Will retry in {} ms...", retryBackoffMs);
- Thread.sleep(retryBackoffMs);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
+ sleepBeforeRetry(attempt);
+ } catch (HugeGraphConnectorException e) {
+ if (!HugeGraphConnectorErrorCode.BUILD_CLIENT_FAILED
+ .getCode()
+ .equals(e.getSeaTunnelErrorCode().getCode())) {
+ throw e;
+ }
+ if (!idempotent) {
+ throw e;
+ }
+ reconnect();
+ if (attempt == totalAttempts) {
throw new HugeGraphConnectorException(
- HugeGraphConnectorErrorCode.OPERATION_RETRY_INTERRUPTED,
- "Graph operation retry was interrupted",
- ie);
+ HugeGraphConnectorErrorCode.BUILD_CLIENT_FAILED,
+ "Failed to establish HugeGraph connection after "
+ + totalAttempts
+ + " attempt(s)",
+ e);
}
-
+ LOG.warn(
+ "HugeGraph connection failed on attempt {}/{}. Error: {}",
+ attempt,
+ totalAttempts,
+ e.getMessage());
+ sleepBeforeRetry(attempt);
} catch (Exception e) {
LOG.error("Non-retryable error executing graph operation: {}", e.getMessage(), e);
throw new HugeGraphConnectorException(
HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
- "Non-retryable error executing graph operation",
+ "Non-retryable error executing graph operation: " + e.getMessage(),
+ e);
+ }
+ }
+ }
+
+ /**
+ * Deterministic 4xx responses (bad request, semantic rejection such as exceeding the server
+ * batch size cap) cannot succeed on retry. Only connection-level failures and 5xx server errors
+ * are worth retrying.
+ */
+ static boolean isRetryable(Exception e) {
+ if (e instanceof ServerException) {
+ int status = ((ServerException) e).status();
+ return status == 408 || status == 425 || status == 429 || status >= 500;
+ }
+ return true;
+ }
+
+ private <T> T executeReadOperation(ReadOperation<T> operation) {
+ int totalAttempts = this.maxRetries + 1;
+ for (int attempt = 1; attempt <= totalAttempts; attempt++) {
+ try {
+ ensureClientInitialized();
+ return operation.execute();
+ } catch (ServerException | ClientException e) {
+ if (!isRetryable(e)) {
+ LOG.error("Server rejected the request (non-retryable): {}", e.getMessage());
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
+ "Server rejected the request (non-retryable): " + e.getMessage(),
+ e);
+ }
+ LOG.warn(
+ "Graph read operation failed on attempt {}/{}. Error: {}",
+ attempt,
+ totalAttempts,
+ e.getMessage());
+ reconnect();
+
+ if (attempt == totalAttempts) {
+ LOG.error("Max retries ({}) reached. Failing task.", this.maxRetries);
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
+ "Failed to execute graph read operation after "
+ + totalAttempts
+ + " attempt(s). Last error: "
+ + e.getMessage(),
+ e);
+ }
+
+ sleepBeforeRetry(attempt);
+ } catch (HugeGraphConnectorException e) {
+ if (!HugeGraphConnectorErrorCode.BUILD_CLIENT_FAILED
+ .getCode()
+ .equals(e.getSeaTunnelErrorCode().getCode())) {
+ throw e;
+ }
+ reconnect();
+ if (attempt == totalAttempts) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.BUILD_CLIENT_FAILED,
+ "Failed to establish HugeGraph connection after "
+ + totalAttempts
+ + " attempt(s)",
+ e);
+ }
+ LOG.warn(
+ "HugeGraph connection failed on attempt {}/{}. Error: {}",
+ attempt,
+ totalAttempts,
+ e.getMessage());
+ sleepBeforeRetry(attempt);
+ } catch (Exception e) {
+ LOG.error(
+ "Non-retryable error executing graph read operation: {}",
+ e.getMessage(),
+ e);
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
+ "Non-retryable error executing graph read operation",
e);
}
}
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
+ "Failed to execute graph read operation");
+ }
+
+ private void sleepBeforeRetry(int attempt) {
+ long delay = computeBackoffMs(retryBackoffMs, retryBackoffMaxMs, attempt);
+ try {
+ LOG.info("Will retry in {} ms (attempt {})...", delay, attempt);
+ Thread.sleep(delay);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.OPERATION_RETRY_INTERRUPTED,
+ "Graph operation retry was interrupted",
+ ie);
+ }
+ }
+
+ /**
+ * Exponential backoff: {@code baseMs * 2^(attempt-1)}, capped at {@code maxMs} (a non-positive
+ * {@code maxMs} means no cap). The shift is bounded so a large {@code maxRetries} cannot
+ * overflow. {@code attempt} is 1-based (the first retry uses the base delay).
+ */
+ static long computeBackoffMs(long baseMs, long maxMs, int attempt) {
+ if (baseMs <= 0) {
+ return 0;
+ }
+ int shift = Math.min(Math.max(attempt - 1, 0), 30);
+ long scaled = baseMs << shift;
+ if (scaled < 0) {
+ // Overflow guard (defensive; the shift cap already prevents this for int-range bases).
+ return maxMs > 0 ? maxMs : Long.MAX_VALUE;
+ }
+ return (maxMs > 0) ? Math.min(scaled, maxMs) : scaled;
}
private SchemaManager getSchema() {
@@ -162,51 +417,483 @@ public final class HugeGraphClient {
return this.schema;
}
+ // --- Schema read operations ---
+
public PropertyKey getPropertyKey(String propertyName) {
- return getSchema().getPropertyKey(propertyName);
+ return executeReadOperation(() -> getSchema().getPropertyKey(propertyName));
}
public VertexLabel getVertexLabel(String label) {
- return getSchema().getVertexLabel(label);
+ VertexLabel vertexLabel = getVertexLabelOrNull(label);
+ if (vertexLabel == null) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ "Vertex label '"
+ + label
+ + "' does not exist in HugeGraph. "
+ + "Please create it first or check your configuration.");
+ }
+ return vertexLabel;
}
public EdgeLabel getEdgeLabel(String label) {
- return getSchema().getEdgeLabel(label);
+ EdgeLabel edgeLabel = getEdgeLabelOrNull(label);
+ if (edgeLabel == null) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ "Edge label '"
+ + label
+ + "' does not exist in HugeGraph. "
+ + "Please create it first or check your configuration.");
+ }
+ return edgeLabel;
}
public String getVertexLabelId(String label) {
- VertexLabel vertexLabel = getSchema().getVertexLabel(label);
+ VertexLabel vertexLabel = getVertexLabel(label);
return String.valueOf(vertexLabel.id());
}
public String getEdgeLabelId(String label) {
- EdgeLabel edgeLabel = getSchema().getEdgeLabel(label);
+ EdgeLabel edgeLabel = getEdgeLabel(label);
return String.valueOf(edgeLabel.id());
}
public IdStrategy getIdStrategy(String label) {
- VertexLabel vertexLabel = getSchema().getVertexLabel(label);
+ VertexLabel vertexLabel = getVertexLabel(label);
return vertexLabel.idStrategy();
}
+ // --- Schema creation operations (idempotent, ifNotExist) ---
+
+ public PropertyKey createPropertyKeyIfNotExist(
+ String name,
+ DataType dataType,
+ org.apache.hugegraph.structure.constant.Cardinality cardinality) {
+ return executeReadOperation(
+ () ->
+ getSchema()
+ .propertyKey(name)
+ .dataType(dataType)
+ .cardinality(cardinality)
+ .ifNotExist()
+ .create());
+ }
+
+ public VertexLabel createVertexLabelIfNotExist(
+ String label,
+ IdStrategy idStrategy,
+ List<String> primaryKeys,
+ List<String> propertyNames,
+ List<String> nullableKeys,
+ LabelOptions options) {
+ return executeReadOperation(
+ () -> {
+ VertexLabel.Builder builder =
+ getSchema().vertexLabel(label).idStrategy(idStrategy);
+ if (idStrategy == IdStrategy.PRIMARY_KEY && primaryKeys != null) {
+ builder.primaryKeys(primaryKeys.toArray(new String[0]));
+ }
+ if (propertyNames != null && !propertyNames.isEmpty()) {
+ builder.properties(propertyNames.toArray(new String[0]));
+ }
+ if (nullableKeys != null && !nullableKeys.isEmpty()) {
+ builder.nullableKeys(nullableKeys.toArray(new String[0]));
+ }
+ if (options != null) {
+ if (options.getTtl() != null && options.getTtl() > 0) {
+ builder.ttl(options.getTtl());
+ if (options.getTtlStartTime() != null
+ && !options.getTtlStartTime().isEmpty()) {
+ builder.ttlStartTime(options.getTtlStartTime());
+ }
+ }
+ if (options.getEnableLabelIndex() != null) {
+ builder.enableLabelIndex(options.getEnableLabelIndex());
+ }
+ if (options.getUserdata() != null) {
+ for (Map.Entry<String, Object> entry :
+ options.getUserdata().entrySet()) {
+ builder.userdata(entry.getKey(), entry.getValue());
+ }
+ }
+ }
+ return builder.ifNotExist().create();
+ });
+ }
+
+ public EdgeLabel createEdgeLabelIfNotExist(
+ String label,
+ String sourceLabel,
+ String targetLabel,
+ Frequency frequency,
+ List<String> sortKeys,
+ List<String> propertyNames,
+ List<String> nullableKeys,
+ LabelOptions options) {
+ return executeReadOperation(
+ () -> {
+ EdgeLabel.Builder builder =
+ getSchema()
+ .edgeLabel(label)
+ .sourceLabel(sourceLabel)
+ .targetLabel(targetLabel);
+ if (frequency != null) {
+ builder.frequency(frequency);
+ }
+ if (sortKeys != null && !sortKeys.isEmpty()) {
+ builder.sortKeys(sortKeys.toArray(new String[0]));
+ }
+ if (propertyNames != null && !propertyNames.isEmpty()) {
+ builder.properties(propertyNames.toArray(new String[0]));
+ }
+ if (nullableKeys != null && !nullableKeys.isEmpty()) {
+ builder.nullableKeys(nullableKeys.toArray(new String[0]));
+ }
+ if (options != null) {
+ if (options.getTtl() != null && options.getTtl() > 0) {
+ builder.ttl(options.getTtl());
+ if (options.getTtlStartTime() != null
+ && !options.getTtlStartTime().isEmpty()) {
+ builder.ttlStartTime(options.getTtlStartTime());
+ }
+ }
+ if (options.getEnableLabelIndex() != null) {
+ builder.enableLabelIndex(options.getEnableLabelIndex());
+ }
+ if (options.getUserdata() != null) {
+ for (Map.Entry<String, Object> entry :
+ options.getUserdata().entrySet()) {
+ builder.userdata(entry.getKey(), entry.getValue());
+ }
+ }
+ }
+ return builder.ifNotExist().create();
+ });
+ }
+
+ /** Check if a property key exists. Returns null if not found. */
+ public PropertyKey getPropertyKeyOrNull(String name) {
+ return executeReadOperation(
+ () -> {
+ try {
+ return getSchema().getPropertyKey(name);
+ } catch (ServerException e) {
+ if (e.status() == 404
+ || (e.getMessage() != null
+ && e.getMessage().contains("does not exist"))) {
+ return null;
+ }
+ throw e;
+ }
+ });
+ }
+
+ /** Check if a vertex label exists. Returns null if not found. */
+ public VertexLabel getVertexLabelOrNull(String label) {
+ return executeReadOperation(
+ () -> {
+ try {
+ return getSchema().getVertexLabel(label);
+ } catch (ServerException e) {
+ if (e.status() == 404
+ || (e.getMessage() != null
+ && e.getMessage().contains("does not exist"))) {
+ return null;
+ }
+ throw e;
+ }
+ });
+ }
+
+ /** Check if an edge label exists. Returns null if not found. */
+ public EdgeLabel getEdgeLabelOrNull(String label) {
+ return executeReadOperation(
+ () -> {
+ try {
+ return getSchema().getEdgeLabel(label);
+ } catch (ServerException e) {
+ if (e.status() == 404
+ || (e.getMessage() != null
+ && e.getMessage().contains("does not exist"))) {
+ return null;
+ }
+ throw e;
+ }
+ });
+ }
+
+ @Override
+ public Set<String> getVertexLabelPropertiesOrNull(String label) {
+ VertexLabel vertexLabel = getVertexLabelOrNull(label);
+ return vertexLabel == null ? null : vertexLabel.properties();
+ }
+
+ @Override
+ public Set<String> getEdgeLabelPropertiesOrNull(String label) {
+ EdgeLabel edgeLabel = getEdgeLabelOrNull(label);
+ return edgeLabel == null ? null : edgeLabel.properties();
+ }
+
+ @Override
+ public List<String> listVertexLabels() {
+ return executeReadOperation(
+ () -> {
+ List<String> names = new ArrayList<>();
+ for (VertexLabel vertexLabel : getSchema().getVertexLabels()) {
+ names.add(vertexLabel.name());
+ }
+ return names;
+ });
+ }
+
+ @Override
+ public List<String> listEdgeLabels() {
+ return executeReadOperation(
+ () -> {
+ List<String> names = new ArrayList<>();
+ for (EdgeLabel edgeLabel : getSchema().getEdgeLabels()) {
+ names.add(edgeLabel.name());
+ }
+ return names;
+ });
+ }
+
+ @Override
+ public DataType getPropertyDataType(String propertyName) {
+ return getPropertyKey(propertyName).dataType();
+ }
+
+ @Override
+ public org.apache.hugegraph.structure.constant.Cardinality getPropertyCardinality(
+ String propertyName) {
+ return getPropertyKey(propertyName).cardinality();
+ }
+
+ // --- Graph write operations ---
+
+ /**
+ * Plain vertex insert — NOT idempotent. A retry after a server-committed-but-client-timed-out
+ * response would create a duplicate. Fails fast on the first error; the caller's single-record
+ * fallback handles the record individually instead.
+ */
public void writeVertex(Vertex vertex) {
- executeGraphOperation(graph -> graph.addVertex(vertex));
+ executeNonIdempotentWrite(graph -> graph.addVertex(vertex));
+ }
+
+ /** Plain edge insert — NOT idempotent. See {@link #writeVertex}. */
+ public void writeEdge(Edge edge, boolean checkVertex) {
+ // Route through addEdges so the single-insert path honors checkVertex the same way the
+ // batch path does (GraphManager.addEdge has no checkVertex overload).
+ executeNonIdempotentWrite(
+ graph -> graph.addEdges(Collections.singletonList(edge), checkVertex));
}
- public void writeEdge(Edge edge) {
- executeGraphOperation(graph -> graph.addEdge(edge));
+ /** Single-vertex property-merge upsert; idempotent — see {@link #batchUpdateVertices}. */
+ public void updateVertex(Vertex vertex, Map<String, UpdateStrategy> updateStrategies) {
+ batchUpdateVertices(Collections.singletonList(vertex), updateStrategies);
}
+ /** Single-edge property-merge upsert; idempotent — see {@link #batchUpdateEdges}. */
+ public void updateEdge(
+ Edge edge, Map<String, UpdateStrategy> updateStrategies, boolean checkVertex) {
+ batchUpdateEdges(Collections.singletonList(edge), updateStrategies, checkVertex);
+ }
+
+ /**
+ * Upserts vertices with per-property merge strategies (OVERRIDE / APPEND / SUM / UNION / ...)
+ * instead of overwriting. Existing vertices are merged; missing ones are created
+ * (createIfNotExist). Idempotent — safe to retry. Chunked like {@link #batchWriteVertices}.
+ */
+ public void batchUpdateVertices(
+ List<Vertex> buffer, Map<String, UpdateStrategy> updateStrategies) {
+ for (int start = 0; start < buffer.size(); start += MAX_RECORDS_PER_BATCH_REQUEST) {
+ List<Vertex> chunk =
+ buffer.subList(
+ start, Math.min(start + MAX_RECORDS_PER_BATCH_REQUEST, buffer.size()));
+ BatchVertexRequest request =
+ new BatchVertexRequest.Builder()
+ .vertices(chunk)
+ .updatingStrategies(updateStrategies)
+ .createIfNotExist(true)
+ .build();
+ executeIdempotentWrite(graph -> graph.updateVertices(request));
+ }
+ }
+
+ /**
+ * Upserts edges with per-property merge strategies. Idempotent. See {@link
+ * #batchUpdateVertices}.
+ */
+ public void batchUpdateEdges(
+ List<Edge> buffer, Map<String, UpdateStrategy> updateStrategies, boolean checkVertex) {
+ for (int start = 0; start < buffer.size(); start += MAX_RECORDS_PER_BATCH_REQUEST) {
+ List<Edge> chunk =
+ buffer.subList(
+ start, Math.min(start + MAX_RECORDS_PER_BATCH_REQUEST, buffer.size()));
+ BatchEdgeRequest request =
+ new BatchEdgeRequest.Builder()
+ .edges(chunk)
+ .updatingStrategies(updateStrategies)
+ .checkVertex(checkVertex)
+ .createIfNotExist(true)
+ .build();
+ executeIdempotentWrite(graph -> graph.updateEdges(request));
+ }
+ }
+
+ /**
+ * Writes vertices in chunks of at most {@link #MAX_RECORDS_PER_BATCH_REQUEST}. The HugeGraph
+ * server rejects batch requests above its per-request cap (default 500, see server option
+ * batch.max_vertices_per_batch), so a user-configured batch_size larger than the cap is split
+ * client-side instead of failing wholesale.
+ *
+ * <p>NOT idempotent — a retry after a server-committed-but-client-timed-out response would
+ * create duplicates. Fails fast; the caller's single-record fallback handles each record.
+ */
+ public void batchWriteVertices(List<Vertex> buffer) {
+ for (int start = 0; start < buffer.size(); start += MAX_RECORDS_PER_BATCH_REQUEST) {
+ List<Vertex> chunk =
+ buffer.subList(
+ start, Math.min(start + MAX_RECORDS_PER_BATCH_REQUEST, buffer.size()));
+ executeNonIdempotentWrite(graph -> graph.addVertices(chunk));
+ }
+ }
+
+ /**
+ * Writes edges in server-cap-sized chunks. NOT idempotent. See {@link #batchWriteVertices} and
+ * {@link #batchWriteEdges}. When {@code checkVertex} is true the server verifies that each
+ * edge's source/target vertices exist, rejecting orphan edges instead of silently writing them
+ * or auto-creating phantom vertices.
+ */
+ public void batchWriteEdges(List<Edge> buffer, boolean checkVertex) {
+ for (int start = 0; start < buffer.size(); start += MAX_RECORDS_PER_BATCH_REQUEST) {
+ List<Edge> chunk =
+ buffer.subList(
+ start, Math.min(start + MAX_RECORDS_PER_BATCH_REQUEST, buffer.size()));
+ executeNonIdempotentWrite(graph -> graph.addEdges(chunk, checkVertex));
+ }
+ }
+
+ // --- Graph read operations ---
+
+ /**
+ * Lists one page of vertices. HugeGraph only enters paged mode when the {@code page} query
+ * parameter is present — a null first page must be sent as an empty string, otherwise the
+ * server returns a single non-paged batch without a next-page marker and the scan silently
+ * stops after {@code limit} records.
+ *
+ * <p>When {@code filter} is non-empty it is passed as the server-side property-equality
+ * condition map, with {@code keepP=true} so the filtered properties are retained in the
+ * returned vertices (they may be part of the output schema).
+ */
+ @SuppressWarnings("unchecked")
+ @Override
+ public PageResult<Vertex> listVertices(
+ String label, Map<String, Object> filter, String page, int limit) {
+ String effectivePage = page == null ? "" : page;
+ boolean hasFilter = filter != null && !filter.isEmpty();
+ Map<String, Object> conditions = hasFilter ? filter : null;
+ return executeReadOperation(
+ () -> {
+ Vertices vertices =
+ this.vertexAPI.list(
+ label, conditions, hasFilter, 0, effectivePage, limit);
+ List<Vertex> records = (List<Vertex>) vertices.results();
+ return new PageResult<>(
+ records == null ? Collections.emptyList() : records, vertices.page());
+ });
+ }
+
+ /** Lists one page of edges. See {@link #listVertices} for the empty-first-page contract. */
+ @SuppressWarnings("unchecked")
+ @Override
+ public PageResult<Edge> listEdges(
+ String label, Map<String, Object> filter, String page, int limit) {
+ String effectivePage = page == null ? "" : page;
+ boolean hasFilter = filter != null && !filter.isEmpty();
+ Map<String, Object> conditions = hasFilter ? filter : null;
+ return executeReadOperation(
+ () -> {
+ Edges edges =
+ this.edgeAPI.list(
+ null,
+ null,
+ label,
+ conditions,
+ hasFilter,
+ 0,
+ effectivePage,
+ limit);
+ List<Edge> records = (List<Edge>) edges.results();
+ return new PageResult<>(
+ records == null ? Collections.emptyList() : records, edges.page());
+ });
+ }
+
+ /**
+ * Splits the vertex keyspace into shards for parallel scanning. Delegates to the server's
+ * {@code traverser().vertexShards} API. Requires a scan-capable backend (RocksDB / HBase /
+ * Cassandra).
+ */
+ @Override
+ public List<Shard> vertexShards(long splitSize) {
+ return executeReadOperation(() -> this.client.traverser().vertexShards(splitSize));
+ }
+
+ /** Splits the edge keyspace into shards. See {@link #vertexShards}. */
+ @Override
+ public List<Shard> edgeShards(long splitSize) {
+ return executeReadOperation(() -> this.client.traverser().edgeShards(splitSize));
+ }
+
+ /**
+ * Scans one page of vertices within {@code shard}. The empty-first-page contract of {@link
+ * #listVertices} applies: a null page is sent as the empty string so the server enters paged
+ * mode. The scan returns vertices of all labels in the key range; label filtering is the
+ * caller's responsibility.
+ */
+ @Override
+ public PageResult<Vertex> scanVertices(Shard shard, String page, int limit) {
+ String effectivePage = page == null ? "" : page;
+ return executeReadOperation(
+ () -> {
+ Vertices vertices =
+ this.client.traverser().vertices(shard, effectivePage, limit);
+ List<Vertex> records = vertices.results();
+ return new PageResult<>(
+ records == null ? Collections.emptyList() : records, vertices.page());
+ });
+ }
+
+ /** Scans one page of edges within {@code shard}. See {@link #scanVertices}. */
+ @Override
+ public PageResult<Edge> scanEdges(Shard shard, String page, int limit) {
+ String effectivePage = page == null ? "" : page;
+ return executeReadOperation(
+ () -> {
+ Edges edges = this.client.traverser().edges(shard, effectivePage, limit);
+ List<Edge> records = edges.results();
+ return new PageResult<>(
+ records == null ? Collections.emptyList() : records, edges.page());
+ });
+ }
+
+ // --- Graph delete operations ---
+
+ /** Delete vertex by id — idempotent (removing an already-deleted vertex is a no-op). */
public void deleteVertex(Object vertexId) {
- executeGraphOperation(graph -> graph.removeVertex(vertexId));
+ executeIdempotentWrite(graph -> graph.removeVertex(vertexId));
}
+ /** Delete edge by id — idempotent. */
public void deleteEdge(String edgeId) {
- executeGraphOperation(graph -> graph.removeEdge(edgeId));
+ executeIdempotentWrite(graph -> graph.removeEdge(edgeId));
}
+ /** Delete vertex with its incident edges — idempotent. */
public void deleteVertexWithEdges(Object vertexId) {
- executeGraphOperation(
+ executeIdempotentWrite(
graph -> {
List<Edge> edges = graph.getEdges(vertexId);
for (Edge edge : edges) {
@@ -216,20 +903,101 @@ public final class HugeGraphClient {
});
}
- public void batchWriteVertices(List<Vertex> buffer) {
- executeGraphOperation(graph -> graph.addVertices(buffer));
+ /**
+ * Returns the names of every edge label whose source or target endpoint is {@code vertexLabel}.
+ * These are the edge labels that would be cascade-deleted if every vertex of {@code
+ * vertexLabel} is removed — used by the DROP_DATA pre-flight safety check.
+ */
+ public List<String> getConnectedEdgeLabels(String vertexLabel) {
+ return executeReadOperation(
+ () -> {
+ List<String> connected = new ArrayList<>();
+ for (EdgeLabel edgeLabel : getSchema().getEdgeLabels()) {
+ if (vertexLabel.equals(edgeLabel.sourceLabel())
+ || vertexLabel.equals(edgeLabel.targetLabel())) {
+ connected.add(edgeLabel.name());
+ }
+ }
+ return connected;
+ });
}
- public void batchWriteEdges(List<Edge> buffer) {
- executeGraphOperation(graph -> graph.addEdges(buffer));
+ /** Page size used when clearing a single label's data for data_save_mode=DROP_DATA. */
+ private static final int DELETE_PAGE_SIZE = 500;
+
+ /**
+ * Deletes every vertex of {@code label} (data only — the VertexLabel schema is preserved), used
+ * by data_save_mode=DROP_DATA to clear just the labels this job targets instead of wiping the
+ * whole graph with {@code clearGraph}. Removing a vertex also removes its incident edges on the
+ * server. Works by repeatedly deleting the first page until none remain, so it does not depend
+ * on a paging cursor staying valid across deletes.
+ */
+ public void deleteVerticesByLabel(String label) {
+ LOG.info("data_save_mode=DROP_DATA: deleting all vertices of label '{}'", label);
+ long deleted = 0;
+ while (true) {
+ List<Vertex> records = listVertices(label, null, "", DELETE_PAGE_SIZE).getRecords();
+ if (records.isEmpty()) {
+ break;
+ }
+ for (Vertex vertex : records) {
+ deleteVertex(vertex.id());
+ deleted++;
+ }
+ }
+ LOG.info("Deleted {} vertices of label '{}'", deleted, label);
}
+ /**
+ * Deletes every edge of {@code label} (data only — the EdgeLabel schema is preserved). See
+ * {@link #deleteVerticesByLabel} for the paging strategy; run before vertex deletion so
+ * edge-only mappings are handled even when their endpoints are out of this job's scope.
+ */
+ public void deleteEdgesByLabel(String label) {
+ LOG.info("data_save_mode=DROP_DATA: deleting all edges of label '{}'", label);
+ long deleted = 0;
+ while (true) {
+ List<Edge> records = listEdges(label, null, "", DELETE_PAGE_SIZE).getRecords();
+ if (records.isEmpty()) {
+ break;
+ }
+ for (Edge edge : records) {
+ deleteEdge(edge.id());
+ deleted++;
+ }
+ }
+ LOG.info("Deleted {} edges of label '{}'", deleted, label);
+ }
+
+ @Override
public void close() {
+ RuntimeException closeFailure = null;
if (this.client != null) {
LOG.info("Closing HugeClient instance.");
- this.client.close();
+ try {
+ this.client.close();
+ } catch (RuntimeException e) {
+ closeFailure = e;
+ }
this.client = null;
- this.schema = null;
+ }
+ if (this.restClient != null) {
+ try {
+ this.restClient.close();
+ } catch (RuntimeException e) {
+ if (closeFailure == null) {
+ closeFailure = e;
+ } else {
+ closeFailure.addSuppressed(e);
+ }
+ }
+ this.restClient = null;
+ }
+ this.vertexAPI = null;
+ this.edgeAPI = null;
+ this.schema = null;
+ if (closeFailure != null) {
+ throw closeFailure;
}
}
}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/client/HugeGraphOperations.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/client/HugeGraphOperations.java
new file mode 100644
index 0000000000..2122f85eb7
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/client/HugeGraphOperations.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.hugegraph.client;
+
+import org.apache.hugegraph.structure.constant.Cardinality;
+import org.apache.hugegraph.structure.constant.DataType;
+import org.apache.hugegraph.structure.graph.Edge;
+import org.apache.hugegraph.structure.graph.Shard;
+import org.apache.hugegraph.structure.graph.Vertex;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public interface HugeGraphOperations {
+
+ Set<String> getVertexLabelPropertiesOrNull(String label);
+
+ Set<String> getEdgeLabelPropertiesOrNull(String label);
+
+ /** Lists the names of all vertex labels defined in the graph schema. */
+ List<String> listVertexLabels();
+
+ /** Lists the names of all edge labels defined in the graph schema. */
+ List<String> listEdgeLabels();
+
+ DataType getPropertyDataType(String propertyName);
+
+ Cardinality getPropertyCardinality(String propertyName);
+
+ /**
+ * Lists one page of vertices of {@code label}. When {@code filter} is non-empty its entries are
+ * applied server-side as property-equality conditions; null/empty means no filtering.
+ */
+ PageResult<Vertex> listVertices(
+ String label, Map<String, Object> filter, String page, int limit);
+
+ /**
+ * Lists one page of edges of {@code label}. See {@link #listVertices} for the filter contract.
+ */
+ PageResult<Edge> listEdges(String label, Map<String, Object> filter, String page, int limit);
+
+ /**
+ * Splits the vertex keyspace into shards of approximately {@code splitSize} bytes each, for
+ * parallel scanning. Requires a backend that supports scan (RocksDB / HBase / Cassandra); the
+ * memory backend does not.
+ */
+ List<Shard> vertexShards(long splitSize);
+
+ /** Splits the edge keyspace into shards. See {@link #vertexShards}. */
+ List<Shard> edgeShards(long splitSize);
+
+ /**
+ * Scans one page of vertices within {@code shard}. Unlike {@link #listVertices}, the scan is by
+ * key range and returns vertices of ALL labels in the range, so the caller must filter by label
+ * client-side; server-side property filters are not supported here.
+ */
+ PageResult<Vertex> scanVertices(Shard shard, String page, int limit);
+
+ /** Scans one page of edges within {@code shard}. See {@link #scanVertices}. */
+ PageResult<Edge> scanEdges(Shard shard, String page, int limit);
+
+ void close();
+}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/MappingConfig.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/client/PageResult.java
similarity index 65%
copy from seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/MappingConfig.java
copy to seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/client/PageResult.java
index c71c2ba914..9124b9e731 100644
--- a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/MappingConfig.java
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/client/PageResult.java
@@ -15,23 +15,15 @@
* limitations under the License.
*/
-package org.apache.seatunnel.connectors.seatunnel.hugegraph.config;
+package org.apache.seatunnel.connectors.seatunnel.hugegraph.client;
import lombok.Data;
-import java.io.Serializable;
import java.util.List;
-import java.util.Map;
@Data
-public class MappingConfig implements Serializable {
- private Map<String, String> fieldMapping;
- private Map<Object, Object> valueMapping;
- private List<String> nullableKeys;
- private List<String> nullValues;
- private List<String> sortKeys;
+public class PageResult<T> {
- // Time config
- private String dateFormat;
- private String timeZone;
+ private final List<T> records;
+ private final String nextPage;
}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphConnectionConfig.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphConnectionConfig.java
new file mode 100644
index 0000000000..0a5eea1adf
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphConnectionConfig.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.hugegraph.config;
+
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorErrorCode;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorException;
+
+import lombok.Data;
+
+import java.io.Serializable;
+
+@Data
+public class HugeGraphConnectionConfig implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private String host;
+ private int port;
+ private String protocol;
+ private String graphName;
+ private String graphSpace;
+ private String username;
+ private String password;
+ private int maxRetries;
+ private int retryBackoffMs;
+ private int retryBackoffMaxMs;
+
+ public static HugeGraphConnectionConfig of(ReadonlyConfig config) {
+ HugeGraphConnectionConfig connectionConfig = new HugeGraphConnectionConfig();
+ connectionConfig.setHost(config.get(HugeGraphOptions.HOST));
+ connectionConfig.setPort(config.get(HugeGraphOptions.PORT));
+ connectionConfig.setProtocol(
+ config.getOptional(HugeGraphOptions.PROTOCOL)
+ .orElse(HugeGraphOptions.PROTOCOL.defaultValue()));
+ connectionConfig.setGraphName(config.get(HugeGraphOptions.GRAPH_NAME));
+ connectionConfig.setGraphSpace(
+ config.getOptional(HugeGraphOptions.GRAPH_SPACE)
+ .filter(graphSpace -> !graphSpace.isEmpty())
+ .orElse(HugeGraphOptions.GRAPH_SPACE.defaultValue()));
+ config.getOptional(HugeGraphOptions.USERNAME).ifPresent(connectionConfig::setUsername);
+ config.getOptional(HugeGraphOptions.PASSWORD).ifPresent(connectionConfig::setPassword);
+ connectionConfig.setMaxRetries(
+ config.getOptional(HugeGraphOptions.MAX_RETRIES)
+ .orElse(HugeGraphOptions.MAX_RETRIES.defaultValue()));
+ connectionConfig.setRetryBackoffMs(
+ config.getOptional(HugeGraphOptions.RETRY_BACKOFF_MS)
+ .orElse(HugeGraphOptions.RETRY_BACKOFF_MS.defaultValue()));
+ connectionConfig.setRetryBackoffMaxMs(
+ config.getOptional(HugeGraphOptions.RETRY_BACKOFF_MAX_MS)
+ .orElse(HugeGraphOptions.RETRY_BACKOFF_MAX_MS.defaultValue()));
+ validate(connectionConfig);
+ return connectionConfig;
+ }
+
+ private static void validate(HugeGraphConnectionConfig config) {
+ // Fail fast at config-load with the offending option name, so the job stops before opening
+ // a client that would otherwise surface a generic connection error much later.
+ if (isBlank(config.getHost())) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ "Option 'host' must not be empty");
+ }
+ if (isBlank(config.getGraphName())) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ "Option 'graph_name' must not be empty");
+ }
+ // graph_space is not validated for emptiness: it carries a non-empty default ("DEFAULT"),
+ // HugeGraphConnectionConfig.of() coalesces blank values to that default, and
+ // HugeGraphClient additionally falls back to "DEFAULT" for any null — so it can never be
+ // empty here.
+ if (config.getPort() < 1 || config.getPort() > 65535) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "Option 'port' must be in range [1, 65535], but got %s",
+ config.getPort()));
+ }
+ if (!"http".equalsIgnoreCase(config.getProtocol())
+ && !"https".equalsIgnoreCase(config.getProtocol())) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "Option 'protocol' must be 'http' or 'https', but got '%s'",
+ config.getProtocol()));
+ }
+ // Credentials must be paired — a lone username or lone password almost always indicates a
+ // config typo and produces a confusing 401 downstream.
+ boolean userSet = !isBlank(config.getUsername());
+ boolean passwordSet = !isBlank(config.getPassword());
+ if (userSet != passwordSet) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ "Options 'username' and 'password' must be set together");
+ }
+ if (config.getMaxRetries() < 0) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ "Option 'max_retries' must be greater than or equal to 0");
+ }
+ if (config.getRetryBackoffMs() < 0) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ "Option 'retry_backoff_ms' must be greater than or equal to 0");
+ }
+ if (config.getRetryBackoffMaxMs() < 0) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ "Option 'retry_backoff_max_ms' must be greater than or equal to 0");
+ }
+ }
+
+ private static boolean isBlank(String value) {
+ return value == null || value.trim().isEmpty();
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/MappingConfig.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphDataSaveMode.java
similarity index 51%
copy from seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/MappingConfig.java
copy to seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphDataSaveMode.java
index c71c2ba914..c8ef5195e0 100644
--- a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/MappingConfig.java
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphDataSaveMode.java
@@ -17,21 +17,21 @@
package org.apache.seatunnel.connectors.seatunnel.hugegraph.config;
-import lombok.Data;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Map;
+/**
+ * Controls how pre-existing data in the target graph is handled before the Sink writes. Mirrors the
+ * SeaTunnel standard {@code DataSaveMode} naming but implemented locally within the connector.
+ */
+public enum HugeGraphDataSaveMode {
-@Data
-public class MappingConfig implements Serializable {
- private Map<String, String> fieldMapping;
- private Map<Object, Object> valueMapping;
- private List<String> nullableKeys;
- private List<String> nullValues;
- private List<String> sortKeys;
+ /** Keep existing data; new elements are written on top (default). */
+ APPEND_DATA,
- // Time config
- private String dateFormat;
- private String timeZone;
+ /**
+ * Before writing, delete the existing data of <em>only the labels this job's mappings
+ * target</em> (edges then vertices), leaving their schema and any other labels' data intact.
+ * Deleting a vertex also removes its incident edges on the server. Scoped per label so, with a
+ * multi-table sink, dropping one table does not wipe another; and on checkpoint restart the
+ * drop is not re-run, so data written before the restart survives.
+ */
+ DROP_DATA
}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphOptions.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphOptions.java
index 5dcaa6c1e0..193f35a357 100644
--- a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphOptions.java
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphOptions.java
@@ -33,6 +33,12 @@ public class HugeGraphOptions {
public static final Option<Integer> PORT =
Options.key("port").intType().noDefaultValue().withDescription("HugeGraph server port");
+ public static final Option<String> PROTOCOL =
+ Options.key("protocol")
+ .stringType()
+ .defaultValue("http")
+ .withDescription("HugeGraph server protocol. Supported values: http, https");
+
public static final Option<String> GRAPH_NAME =
Options.key("graph_name")
.stringType()
@@ -42,8 +48,9 @@ public class HugeGraphOptions {
public static final Option<String> GRAPH_SPACE =
Options.key("graph_space")
.stringType()
- .noDefaultValue()
- .withDescription("The graph space of the graph to be operated on");
+ .defaultValue("DEFAULT")
+ .withDescription(
+ "The graph space the graph belongs to. Defaults to 'DEFAULT'.");
public static final Option<String> USERNAME =
Options.key("username")
@@ -66,6 +73,52 @@ public class HugeGraphOptions {
.defaultValue(5000)
.withDescription("The batch flash period");
+ public static final Option<Boolean> CHECK_VERTEX =
+ Options.key("check_vertex")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "Whether the server verifies that an edge's source/target vertices "
+ + "exist when writing edges. When false (default), edges whose "
+ + "endpoints were never loaded are written as orphan edges (or "
+ + "trigger server-side phantom vertex auto-creation). Enable to "
+ + "reject such edges.");
+
+ public static final Option<Boolean> BATCH_FAILURE_FALLBACK =
+ Options.key("batch_failure_fallback")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "When true, a failed batch insert falls back to inserting records one "
+ + "by one so a single bad ('poison') record no longer fails the "
+ + "whole batch. Failed records are logged and skipped; the rest "
+ + "succeed. Default false (fail-fast): any batch failure fails "
+ + "the task immediately. Opt in explicitly when record skipping "
+ + "is acceptable.");
+
+ public static final Option<Integer> MAX_INSERT_ERRORS =
+ Options.key("max_insert_errors")
+ .intType()
+ .defaultValue(0)
+ .withDescription(
+ "Maximum number of records that may be skipped by the single-record "
+ + "fallback (batch_failure_fallback=true) before the task is "
+ + "failed. Default 0: any skipped record fails the task. Set to "
+ + "-1 for unlimited (never fail on skipped records). Only applies "
+ + "when batch_failure_fallback is enabled.");
+
+ public static final Option<String> FAILURE_DATA_PATH =
+ Options.key("failure_data_path")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "Optional local directory. When set, every record skipped by the "
+ + "single-record fallback is appended (as the mapped vertex/edge "
+ + "id, label, properties and the server error) to a per-subtask "
+ + "file under this directory for offline investigation. Note: in "
+ + "cluster mode the file is created on the worker node running "
+ + "the sink subtask, not the submitting client.");
+
public static final Option<Integer> MAX_RETRIES =
Options.key("max_retries").intType().defaultValue(3).withDescription("The retry times");
@@ -73,5 +126,16 @@ public class HugeGraphOptions {
Options.key("retry_backoff_ms")
.intType()
.defaultValue(5000)
- .withDescription("The retry backoff time");
+ .withDescription(
+ "The base retry backoff time in milliseconds. Backoff grows "
+ + "exponentially per attempt (retry_backoff_ms * 2^(attempt-1)), "
+ + "capped at retry_backoff_max_ms.");
+
+ public static final Option<Integer> RETRY_BACKOFF_MAX_MS =
+ Options.key("retry_backoff_max_ms")
+ .intType()
+ .defaultValue(30000)
+ .withDescription(
+ "Upper bound in milliseconds for the exponential retry backoff, so a "
+ + "high max_retries cannot produce pathologically long sleeps.");
}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/MappingConfig.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphSchemaSaveMode.java
similarity index 64%
copy from seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/MappingConfig.java
copy to seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphSchemaSaveMode.java
index c71c2ba914..0dbb565aa0 100644
--- a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/MappingConfig.java
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphSchemaSaveMode.java
@@ -17,21 +17,15 @@
package org.apache.seatunnel.connectors.seatunnel.hugegraph.config;
-import lombok.Data;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Map;
+/**
+ * Controls schema management behavior during Sink initialization. Aligns with SeaTunnel standard
+ * {@code SchemaSaveMode} naming but implemented locally within the connector.
+ */
+public enum HugeGraphSchemaSaveMode {
-@Data
-public class MappingConfig implements Serializable {
- private Map<String, String> fieldMapping;
- private Map<Object, Object> valueMapping;
- private List<String> nullableKeys;
- private List<String> nullValues;
- private List<String> sortKeys;
+ /** Auto-create missing PropertyKey/VertexLabel/EdgeLabel; never modify existing schema. */
+ CREATE_SCHEMA_WHEN_NOT_EXIST,
- // Time config
- private String dateFormat;
- private String timeZone;
+ /** Do not create any schema; fail immediately if schema is missing or mismatched. */
+ ERROR_WHEN_SCHEMA_NOT_EXIST
}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphSinkConfig.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphSinkConfig.java
index dcb0fb32c3..398423ec07 100644
--- a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphSinkConfig.java
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphSinkConfig.java
@@ -18,60 +18,241 @@
package org.apache.seatunnel.connectors.seatunnel.hugegraph.config;
import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorErrorCode;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import lombok.Data;
import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
@Data
public class HugeGraphSinkConfig implements Serializable {
- private String host;
- private int port;
- private String graphName;
- private String graphSpace;
- private String username;
- private String password;
- private SchemaConfig schemaConfig;
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(HugeGraphSinkConfig.class);
+
+ // Shared connection config
+ private HugeGraphConnectionConfig connectionConfig;
+
+ // Batch config
private int batchSize;
private int batchIntervalMs;
+ private boolean batchFailureFallback;
+ private boolean checkVertex;
private int maxRetries;
private int retryBackoffMs;
+ // Max records the single-record fallback may skip before the task fails (-1 = unlimited).
+ private int maxInsertErrors;
+ // Optional directory to persist skipped-record failure samples; null = do not persist.
+ private String failureDataPath;
+
+ // New: multi-mapping config
+ private List<MappingConfig> mappings;
+ private HugeGraphSchemaSaveMode schemaSaveMode;
+ private HugeGraphDataSaveMode dataSaveMode;
+ private boolean deleteVertexWithEdges;
+ private boolean allowCascadeDeleteUnmappedEdges;
- // mapping config
+ // Legacy (deprecated, kept for backward compat parsing only)
+ private SchemaConfig schemaConfig;
private List<String> selectedFields;
private List<String> ignoredFields;
public static HugeGraphSinkConfig of(ReadonlyConfig config) {
HugeGraphSinkConfig sinkConfig = new HugeGraphSinkConfig();
- sinkConfig.setHost(config.get(HugeGraphOptions.HOST));
- sinkConfig.setPort(config.get(HugeGraphOptions.PORT));
- sinkConfig.setGraphName(config.get(HugeGraphOptions.GRAPH_NAME));
+ // Connection
+ sinkConfig.setConnectionConfig(HugeGraphConnectionConfig.of(config));
+
+ // Batch
sinkConfig.setBatchSize(
config.getOptional(HugeGraphOptions.BATCH_SIZE)
.orElse(HugeGraphOptions.BATCH_SIZE.defaultValue()));
sinkConfig.setBatchIntervalMs(
config.getOptional(HugeGraphOptions.BATCH_INTERVAL_MS)
.orElse(HugeGraphOptions.BATCH_INTERVAL_MS.defaultValue()));
- sinkConfig.setMaxRetries(
- config.getOptional(HugeGraphOptions.MAX_RETRIES)
- .orElse(HugeGraphOptions.MAX_RETRIES.defaultValue()));
- sinkConfig.setRetryBackoffMs(
- config.getOptional(HugeGraphOptions.RETRY_BACKOFF_MS)
- .orElse(HugeGraphOptions.RETRY_BACKOFF_MS.defaultValue()));
- sinkConfig.setSchemaConfig(config.get(HugeGraphSinkOptions.SCHEMA_CONFIG));
+ sinkConfig.setBatchFailureFallback(
+ config.getOptional(HugeGraphOptions.BATCH_FAILURE_FALLBACK)
+ .orElse(HugeGraphOptions.BATCH_FAILURE_FALLBACK.defaultValue()));
+ sinkConfig.setCheckVertex(
+ config.getOptional(HugeGraphOptions.CHECK_VERTEX)
+ .orElse(HugeGraphOptions.CHECK_VERTEX.defaultValue()));
+ sinkConfig.setMaxRetries(sinkConfig.getConnectionConfig().getMaxRetries());
+ sinkConfig.setRetryBackoffMs(sinkConfig.getConnectionConfig().getRetryBackoffMs());
+ sinkConfig.setMaxInsertErrors(
+ config.getOptional(HugeGraphOptions.MAX_INSERT_ERRORS)
+ .orElse(HugeGraphOptions.MAX_INSERT_ERRORS.defaultValue()));
+ config.getOptional(HugeGraphOptions.FAILURE_DATA_PATH)
+ .ifPresent(sinkConfig::setFailureDataPath);
+
+ // Resolve mappings with backward compatibility
+ sinkConfig.setMappings(resolveMappings(config, sinkConfig));
+ applyMappingDefaults(sinkConfig.getMappings());
+
+ // Multi-table contract: source_table is an ALL-or-NOTHING switch.
+ // All absent → single-table backward-compatible (each mapping activates in the one writer).
+ // All present → multi-table (each mapping activates only in its matching writer).
+ // Mixed → misconfiguration; fail fast with a clear diagnostic.
+ if (sinkConfig.getMappings() != null) {
+ validateSourceTableConsistency(sinkConfig.getMappings());
+ }
+ boolean legacyConfig = sinkConfig.getSchemaConfig() != null;
+ sinkConfig.setSchemaSaveMode(
+ config.getOptional(HugeGraphSinkOptions.SCHEMA_SAVE_MODE)
+ .orElse(
+ legacyConfig
+ ? HugeGraphSchemaSaveMode.ERROR_WHEN_SCHEMA_NOT_EXIST
+ : HugeGraphSinkOptions.SCHEMA_SAVE_MODE.defaultValue()));
+ sinkConfig.setDeleteVertexWithEdges(
+ config.getOptional(HugeGraphSinkOptions.DELETE_VERTEX_WITH_EDGES)
+ .orElse(
+ legacyConfig
+ ? true
+ : HugeGraphSinkOptions.DELETE_VERTEX_WITH_EDGES
+ .defaultValue()));
+ sinkConfig.setAllowCascadeDeleteUnmappedEdges(
+ config.getOptional(HugeGraphSinkOptions.ALLOW_CASCADE_DELETE_UNMAPPED_EDGES)
+ .orElse(
+ HugeGraphSinkOptions.ALLOW_CASCADE_DELETE_UNMAPPED_EDGES
+ .defaultValue()));
+ sinkConfig.setDataSaveMode(
+ config.getOptional(HugeGraphSinkOptions.DATA_SAVE_MODE)
+ .orElse(HugeGraphSinkOptions.DATA_SAVE_MODE.defaultValue()));
+
+ // Deprecated fields (parse but warn)
config.getOptional(HugeGraphSinkOptions.SELECTED_FIELDS)
- .ifPresent(sinkConfig::setSelectedFields);
+ .ifPresent(
+ fields -> {
+ LOG.warn(
+ "Option 'selected_fields' is deprecated. Use 'properties' within each mapping instead.");
+ sinkConfig.setSelectedFields(fields);
+ });
config.getOptional(HugeGraphSinkOptions.IGNORED_FIELDS)
- .ifPresent(sinkConfig::setIgnoredFields);
-
- config.getOptional(HugeGraphOptions.GRAPH_SPACE).ifPresent(sinkConfig::setGraphSpace);
- config.getOptional(HugeGraphOptions.USERNAME).ifPresent(sinkConfig::setUsername);
- config.getOptional(HugeGraphOptions.PASSWORD).ifPresent(sinkConfig::setPassword);
+ .ifPresent(
+ fields -> {
+ LOG.warn(
+ "Option 'ignored_fields' is deprecated. Use 'properties' within each mapping instead.");
+ sinkConfig.setIgnoredFields(fields);
+ });
return sinkConfig;
}
+
+ /**
+ * Converts legacy global field selection into the mapping property list. The old writer ignored
+ * {@code schema_config.properties} and wrote all fields after applying selected/ignored fields,
+ * so preserving that behavior requires the input row schema.
+ */
+ public void applyLegacyFieldSelection(SeaTunnelRowType rowType) {
+ if (schemaConfig == null || mappings == null || mappings.isEmpty()) {
+ return;
+ }
+
+ List<String> effectiveFields;
+ if (selectedFields != null && !selectedFields.isEmpty()) {
+ effectiveFields = new ArrayList<>(selectedFields);
+ } else {
+ effectiveFields = new ArrayList<>(Arrays.asList(rowType.getFieldNames()));
+ if (ignoredFields != null && !ignoredFields.isEmpty()) {
+ Set<String> ignored = new HashSet<>(ignoredFields);
+ effectiveFields.removeIf(ignored::contains);
+ }
+ }
+ mappings.get(0).setProperties(effectiveFields);
+ }
+
+ private static List<MappingConfig> resolveMappings(
+ ReadonlyConfig config, HugeGraphSinkConfig sinkConfig) {
+ boolean hasMappings = config.getOptional(HugeGraphSinkOptions.MAPPINGS).isPresent();
+ boolean hasSchemaConfig =
+ config.getOptional(HugeGraphSinkOptions.SCHEMA_CONFIG).isPresent();
+
+ if (hasMappings) {
+ if (hasSchemaConfig) {
+ LOG.warn(
+ "Both 'mappings' and 'schema_config' are present. "
+ + "'schema_config' will be ignored. Please migrate to 'mappings'.");
+ }
+ return config.get(HugeGraphSinkOptions.MAPPINGS);
+ }
+
+ if (hasSchemaConfig) {
+ SchemaConfig schemaConfig = config.get(HugeGraphSinkOptions.SCHEMA_CONFIG);
+ sinkConfig.setSchemaConfig(schemaConfig);
+ return Collections.singletonList(MappingConfig.fromLegacySchemaConfig(schemaConfig));
+ }
+
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ "Either 'mappings' or 'schema_config' must be specified. "
+ + "'mappings' is the recommended option.");
+ }
+
+ private static void applyMappingDefaults(List<MappingConfig> mappings) {
+ if (mappings == null) {
+ return;
+ }
+ for (MappingConfig m : mappings) {
+ if (m.getDateFormat() == null || m.getDateFormat().isEmpty()) {
+ m.setDateFormat("yyyy-MM-dd");
+ }
+ // Leave timeZone unset when the user did not configure one; DataTypeUtil then falls
+ // back to ZoneId.systemDefault(), matching the HugeGraph Source. Hard-coding GMT+8
+ // here previously silently shifted absolute times by up to 8 hours when the Source
+ // ran on a JVM whose default zone was not Asia/Shanghai.
+ }
+ }
+
+ /**
+ * Enforces the ALL-or-NOTHING contract on {@code source_table}.
+ *
+ * <p>All mappings with {@code source_table} set → multi-table mode: each mapping activates only
+ * in the writer whose {@code CatalogTable.getTablePath()} matches. All mappings without {@code
+ * source_table} → single-table backward-compatible: every mapping activates in the one writer.
+ * A mix of set and unset is ambiguous — the user either forgot to add {@code source_table} to
+ * some mappings, or accidentally added it to one. Refuse with a clear diagnostic.
+ */
+ static void validateSourceTableConsistency(List<MappingConfig> mappings) {
+ boolean anySet = false;
+ boolean anyUnset = false;
+ List<String> setLabels = new ArrayList<>();
+ List<String> unsetLabels = new ArrayList<>();
+
+ for (MappingConfig m : mappings) {
+ if (m.getSourceTable() == null || m.getSourceTable().isEmpty()) {
+ anyUnset = true;
+ unsetLabels.add(m.getLabel());
+ } else {
+ anySet = true;
+ setLabels.add(m.getLabel());
+ }
+ }
+
+ if (anySet && anyUnset) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "Inconsistent 'source_table' configuration. %d mapping(s) set it (%s), "
+ + "but %d mapping(s) are missing it (%s). "
+ + "'source_table' is an ALL-or-NOTHING switch: either every "
+ + "mapping declares it (multi-table mode — each mapping activates "
+ + "only in the matching writer), or none do (single-table mode — "
+ + "the backward-compatible default). Check that you haven't "
+ + "forgotten 'source_table' on some mappings, or added it to one "
+ + "mapping by mistake.",
+ setLabels.size(), setLabels, unsetLabels.size(), unsetLabels));
+ }
+ }
}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphSinkOptions.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphSinkOptions.java
index 815ca78928..e2c1b874ff 100644
--- a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphSinkOptions.java
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphSinkOptions.java
@@ -24,22 +24,73 @@ import java.util.List;
public class HugeGraphSinkOptions {
+ public static final Option<List<MappingConfig>> MAPPINGS =
+ Options.key("mappings")
+ .listType(MappingConfig.class)
+ .noDefaultValue()
+ .withDescription(
+ "List of mapping configurations. Each mapping describes how to write "
+ + "a single vertex or edge label from the input row.");
+
+ public static final Option<HugeGraphSchemaSaveMode> SCHEMA_SAVE_MODE =
+ Options.key("schema_save_mode")
+ .enumType(HugeGraphSchemaSaveMode.class)
+ .defaultValue(HugeGraphSchemaSaveMode.CREATE_SCHEMA_WHEN_NOT_EXIST)
+ .withDescription(
+ "Schema management mode. CREATE_SCHEMA_WHEN_NOT_EXIST (default) auto-creates "
+ + "missing PropertyKey/VertexLabel/EdgeLabel. ERROR_WHEN_SCHEMA_NOT_EXIST "
+ + "fails if schema is missing.");
+
+ public static final Option<HugeGraphDataSaveMode> DATA_SAVE_MODE =
+ Options.key("data_save_mode")
+ .enumType(HugeGraphDataSaveMode.class)
+ .defaultValue(HugeGraphDataSaveMode.APPEND_DATA)
+ .withDescription(
+ "How pre-existing data is handled before writing. APPEND_DATA (default) "
+ + "keeps existing data. DROP_DATA deletes the existing data of only "
+ + "the labels this job targets (edges then vertices) at job start, "
+ + "preserving their schema and any other labels' data; the drop is "
+ + "scoped per label and is not re-run on checkpoint restart.");
+
+ public static final Option<Boolean> DELETE_VERTEX_WITH_EDGES =
+ Options.key("delete_vertex_with_edges")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "When true, DELETE rows for vertices will cascade-delete associated edges. "
+ + "Default false: only the vertex itself is deleted.");
+
+ public static final Option<Boolean> ALLOW_CASCADE_DELETE_UNMAPPED_EDGES =
+ Options.key("allow_cascade_delete_unmapped_edges")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "When data_save_mode is DROP_DATA, deleting vertices cascades to their "
+ + "incident edges — including edge labels not listed in this job's "
+ + "mappings. Default false: the job fails fast and lists the unmapped "
+ + "edge labels. Set to true to accept the destructive cascade.");
+
+ // --- Legacy options (deprecated, kept for backward compatibility) ---
+
+ public static final Option<SchemaConfig> SCHEMA_CONFIG =
+ Options.key("schema_config")
+ .objectType(SchemaConfig.class)
+ .noDefaultValue()
+ .withDescription(
+ "[Deprecated] Use 'mappings' instead. Legacy schema configuration object "
+ + "that describes the mapping to a vertex or edge.");
+
public static final Option<List<String>> SELECTED_FIELDS =
Options.key("selected_fields")
.listType()
.noDefaultValue()
- .withDescription("Selected Fields");
+ .withDescription(
+ "[Deprecated] Use 'properties' within each mapping instead. Selected fields.");
public static final Option<List<String>> IGNORED_FIELDS =
Options.key("ignored_fields")
.listType()
.noDefaultValue()
- .withDescription("Ignored Fields");
-
- public static final Option<SchemaConfig> SCHEMA_CONFIG =
- Options.key("schema_config")
- .objectType(SchemaConfig.class)
- .noDefaultValue()
.withDescription(
- "Schema configuration object that describes the mapping to a vertex or edge.");
+ "[Deprecated] Use 'properties' within each mapping instead. Ignored fields.");
}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphSourceConfig.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphSourceConfig.java
new file mode 100644
index 0000000000..0d6d3bcb02
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphSourceConfig.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.hugegraph.config;
+
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorErrorCode;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorException;
+
+import lombok.Data;
+
+import java.io.Serializable;
+import java.time.DateTimeException;
+import java.time.ZoneId;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+@Data
+public class HugeGraphSourceConfig implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private HugeGraphConnectionConfig connectionConfig;
+ private String label;
+ private MappingConfig.LabelType labelType;
+ // Read-all-labels mode: when true, {@code label}/{@code schema} are null and {@code labels}
+ // holds every label of {@code labelType} to read (one produced table each).
+ private boolean readAllLabels;
+ private List<String> labels;
+ private SeaTunnelRowType schema;
+ private int pageSize;
+ private long splitSize;
+ private String timeZone;
+ // Optional server-side property equality conditions; null/empty = read all elements.
+ private Map<String, Object> filter;
+
+ public static HugeGraphSourceConfig of(ReadonlyConfig config, SeaTunnelRowType schema) {
+ HugeGraphSourceConfig sourceConfig = new HugeGraphSourceConfig();
+ sourceConfig.setConnectionConfig(HugeGraphConnectionConfig.of(config));
+ sourceConfig.setReadAllLabels(false);
+ sourceConfig.setLabel(config.get(HugeGraphSourceOptions.LABEL));
+ sourceConfig.setLabels(Collections.singletonList(config.get(HugeGraphSourceOptions.LABEL)));
+ sourceConfig.setLabelType(
+ config.getOptional(HugeGraphSourceOptions.LABEL_TYPE)
+ .orElse(HugeGraphSourceOptions.LABEL_TYPE.defaultValue()));
+ sourceConfig.setSchema(schema);
+ sourceConfig.setPageSize(
+ config.getOptional(HugeGraphSourceOptions.PAGE_SIZE)
+ .orElse(HugeGraphSourceOptions.PAGE_SIZE.defaultValue()));
+ sourceConfig.setSplitSize(
+ config.getOptional(HugeGraphSourceOptions.SPLIT_SIZE)
+ .orElse(HugeGraphSourceOptions.SPLIT_SIZE.defaultValue()));
+ config.getOptional(HugeGraphSourceOptions.TIME_ZONE).ifPresent(sourceConfig::setTimeZone);
+ config.getOptional(HugeGraphSourceOptions.FILTER).ifPresent(sourceConfig::setFilter);
+ validate(sourceConfig);
+ return sourceConfig;
+ }
+
+ /**
+ * Read-all-labels construction: no single {@code label} and no user {@code schema}/{@code
+ * filter}; the labels are discovered from the server and each gets its own auto-discovered row
+ * type. See {@link
+ * org.apache.seatunnel.connectors.seatunnel.hugegraph.source.HugeGraphSourceFactory}.
+ */
+ public static HugeGraphSourceConfig ofReadAll(ReadonlyConfig config, List<String> labels) {
+ HugeGraphSourceConfig sourceConfig = new HugeGraphSourceConfig();
+ sourceConfig.setConnectionConfig(HugeGraphConnectionConfig.of(config));
+ sourceConfig.setReadAllLabels(true);
+ sourceConfig.setLabel(null);
+ sourceConfig.setLabels(labels);
+ sourceConfig.setLabelType(
+ config.getOptional(HugeGraphSourceOptions.LABEL_TYPE)
+ .orElse(HugeGraphSourceOptions.LABEL_TYPE.defaultValue()));
+ sourceConfig.setSchema(null);
+ sourceConfig.setPageSize(
+ config.getOptional(HugeGraphSourceOptions.PAGE_SIZE)
+ .orElse(HugeGraphSourceOptions.PAGE_SIZE.defaultValue()));
+ sourceConfig.setSplitSize(
+ config.getOptional(HugeGraphSourceOptions.SPLIT_SIZE)
+ .orElse(HugeGraphSourceOptions.SPLIT_SIZE.defaultValue()));
+ config.getOptional(HugeGraphSourceOptions.TIME_ZONE).ifPresent(sourceConfig::setTimeZone);
+ validate(sourceConfig);
+ return sourceConfig;
+ }
+
+ private static void validate(HugeGraphSourceConfig sourceConfig) {
+ int pageSize = sourceConfig.getPageSize();
+ if (pageSize < HugeGraphSourceOptions.MIN_PAGE_SIZE
+ || pageSize > HugeGraphSourceOptions.MAX_PAGE_SIZE) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "Option 'page_size' must be in range [%s, %s], but got %s",
+ HugeGraphSourceOptions.MIN_PAGE_SIZE,
+ HugeGraphSourceOptions.MAX_PAGE_SIZE,
+ pageSize));
+ }
+
+ if (sourceConfig.getSplitSize() < HugeGraphSourceOptions.MIN_SPLIT_SIZE) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "Option 'split_size' must be at least %s bytes (the HugeGraph minimum "
+ + "shard size); a smaller value would split the keyspace into a "
+ + "huge number of shards and risk OOM / oversized checkpoints. "
+ + "Got %s.",
+ HugeGraphSourceOptions.MIN_SPLIT_SIZE, sourceConfig.getSplitSize()));
+ }
+
+ if (sourceConfig.isReadAllLabels()) {
+ // Read-all mode discovers labels from the server; there must be at least one, and no
+ // single user schema applies (each label gets its own auto-discovered row type).
+ if (sourceConfig.getLabels() == null || sourceConfig.getLabels().isEmpty()) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ "Read-all-labels mode requires at least one label, but none were discovered.");
+ }
+ } else if (sourceConfig.getSchema() == null) {
+ // Single-label mode: schema must be present, but an empty fields block is valid — a
+ // property-less label (e.g. a pure relationship edge, or a vertex with no properties)
+ // is exported as just the reserved columns (~id/~label/…). Requiring a fake property
+ // would make such labels unreadable.
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ "Option 'schema' is required (use 'schema = { fields {} }' for a label with no properties)");
+ }
+ if (sourceConfig.getTimeZone() != null) {
+ try {
+ ZoneId.of(sourceConfig.getTimeZone());
+ } catch (DateTimeException e) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "Option 'time_zone' must be a valid ZoneId, but got '%s'",
+ sourceConfig.getTimeZone()),
+ e);
+ }
+ }
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphSourceOptions.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphSourceOptions.java
new file mode 100644
index 0000000000..5a42f1b79a
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/HugeGraphSourceOptions.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.hugegraph.config;
+
+import org.apache.seatunnel.shade.com.fasterxml.jackson.core.type.TypeReference;
+
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+
+import java.util.Map;
+
+public class HugeGraphSourceOptions {
+
+ public static final int MIN_PAGE_SIZE = 100;
+ public static final int MAX_PAGE_SIZE = 10000;
+ // Lower bound for split_size (1 MiB), matching the HugeGraph server's own minimum shard size.
+ // A smaller value shatters the keyspace into a huge number of shards — one split per shard,
+ // each
+ // persisted into every checkpoint — risking OOM / oversized checkpoints, and the server rejects
+ // it anyway; reject it up front with a clear message.
+ public static final long MIN_SPLIT_SIZE = 1048576L;
+
+ public static final Option<String> LABEL =
+ Options.key("label")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("HugeGraph vertex label or edge label to read");
+
+ public static final Option<MappingConfig.LabelType> LABEL_TYPE =
+ Options.key("label_type")
+ .enumType(MappingConfig.LabelType.class)
+ .defaultValue(MappingConfig.LabelType.VERTEX)
+ .withDescription("HugeGraph label type. Supported values are VERTEX and EDGE");
+
+ public static final Option<Integer> PAGE_SIZE =
+ Options.key("page_size")
+ .intType()
+ .defaultValue(1000)
+ .withDescription("Records per HugeGraph page, must be in range [100, 10000]");
+
+ public static final Option<String> TIME_ZONE =
+ Options.key("time_zone")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "Time zone used to convert HugeGraph DATE values that the server returns "
+ + "as an epoch/Date (the instant is rendered as a local date-time "
+ + "in this zone). It does NOT apply when the server returns a DATE "
+ + "already serialized as a wall-clock string (e.g. "
+ + "'yyyy-MM-dd HH:mm:ss.SSS') — that value is kept verbatim, since "
+ + "its original zone is not carried in the string. When omitted, "
+ + "the worker JVM default time zone is used for backward "
+ + "compatibility.");
+
+ public static final Option<Long> SPLIT_SIZE =
+ Options.key("split_size")
+ .longType()
+ .defaultValue(1048576L)
+ .withDescription(
+ "Target size in bytes of each key-range shard when parallelism > 1. "
+ + "The server splits the keyspace into shards of roughly this "
+ + "size and readers scan them in parallel; a larger value yields "
+ + "fewer, bigger shards. Ignored when parallelism = 1 (which uses "
+ + "the single label-list scan). Requires a scan-capable backend "
+ + "(RocksDB / HBase / Cassandra).");
+
+ public static final Option<Map<String, Object>> FILTER =
+ Options.key("filter")
+ .type(new TypeReference<Map<String, Object>>() {})
+ .noDefaultValue()
+ .withDescription(
+ "Optional property equality conditions applied server-side when "
+ + "reading the label, e.g. { country = \"US\", active = \"true\" }. "
+ + "Only elements whose properties match all entries are returned. "
+ + "Every key must be a property of the configured label. When "
+ + "omitted, all elements of the label are read.");
+}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/LabelOptions.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/LabelOptions.java
new file mode 100644
index 0000000000..77c7d4cb01
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/LabelOptions.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.hugegraph.config;
+
+import java.util.Map;
+
+/**
+ * Optional HugeGraph label attributes applied at schema-creation time (TTL, TTL start-time
+ * property, label-index toggle, and user-defined metadata). All fields are nullable; a null field
+ * means "leave at the HugeGraph server default".
+ */
+public class LabelOptions {
+
+ private final Long ttl;
+ private final String ttlStartTime;
+ private final Boolean enableLabelIndex;
+ private final Map<String, Object> userdata;
+
+ public LabelOptions(
+ Long ttl, String ttlStartTime, Boolean enableLabelIndex, Map<String, Object> userdata) {
+ this.ttl = ttl;
+ this.ttlStartTime = ttlStartTime;
+ this.enableLabelIndex = enableLabelIndex;
+ this.userdata = userdata;
+ }
+
+ public Long getTtl() {
+ return ttl;
+ }
+
+ public String getTtlStartTime() {
+ return ttlStartTime;
+ }
+
+ public Boolean getEnableLabelIndex() {
+ return enableLabelIndex;
+ }
+
+ public Map<String, Object> getUserdata() {
+ return userdata;
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/ListFormat.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/ListFormat.java
new file mode 100644
index 0000000000..35aae38946
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/ListFormat.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.hugegraph.config;
+
+import lombok.Data;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * How a raw string cell is parsed into the elements of a SET / LIST property. Defaults preserve the
+ * connector's historical behavior: an optional surrounding {@code [ ]}, comma-separated elements,
+ * with blank elements dropped.
+ */
+@Data
+public class ListFormat implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ /** Optional leading symbol stripped before splitting (e.g. {@code [}). Empty disables it. */
+ private String startSymbol = "[";
+
+ /** Optional trailing symbol stripped before splitting (e.g. {@code ]}). Empty disables it. */
+ private String endSymbol = "]";
+
+ /** Delimiter between elements. */
+ private String elemDelimiter = ",";
+
+ /** Element values to drop after splitting (in addition to blank elements). */
+ private List<String> ignoredElems;
+
+ public List<String> getIgnoredElems() {
+ return ignoredElems == null ? Collections.emptyList() : ignoredElems;
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/MappingConfig.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/MappingConfig.java
index c71c2ba914..c25649f485 100644
--- a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/MappingConfig.java
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/MappingConfig.java
@@ -17,21 +17,208 @@
package org.apache.seatunnel.connectors.seatunnel.hugegraph.config;
+import org.apache.hugegraph.structure.constant.Frequency;
+import org.apache.hugegraph.structure.constant.IdStrategy;
+import org.apache.hugegraph.structure.graph.UpdateStrategy;
+
import lombok.Data;
import java.io.Serializable;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
@Data
public class MappingConfig implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ // Element type
+ private LabelType type;
+ private String label;
+
+ // Optional: binds this mapping to a specific input CatalogTable.
+ // When set, the mapping only activates in a writer whose tablePath.toString()
+ // matches (multi-table sink). When absent / empty, the mapping activates in
+ // every writer — backward compatible with single-table jobs where there is
+ // only one writer. The value should be the table path string as it appears
+ // in the source's produced CatalogTable (e.g. "hugegraph.person").
+ private String sourceTable;
+
+ // Vertex-specific
+ private IdStrategy idStrategy;
+ private List<String> idFields;
+ // Expand a list-valued id cell into one vertex per element (INSERT/append only, CUSTOMIZE ids).
+ private boolean unfold;
+
+ // Edge-specific
+ private SourceTargetConfig sourceConfig;
+ private SourceTargetConfig targetConfig;
+ private Frequency frequency;
+ private List<String> sortKeys;
+ // Expand a list-valued source/target id cell into multiple edges (cartesian; INSERT/append
+ // only,
+ // CUSTOMIZE endpoint ids).
+ private boolean unfoldSource;
+ private boolean unfoldTarget;
+
+ // Property config. `properties` is the selected whitelist (only these source fields become
+ // properties); when empty, all input fields are used. `ignored` is the opposite blacklist
+ // (all fields except these). The two are mutually exclusive.
+ private List<String> properties;
+ private List<String> ignored;
+
+ // Field mapping (source field name → target property name)
private Map<String, String> fieldMapping;
- private Map<Object, Object> valueMapping;
+ // Per-field value mapping: outer key = source field name, inner map = rawValue -> mappedValue.
+ // Scoping by field prevents one column's rule from bleeding into another (e.g. gender M->male
+ // must not also rewrite status M).
+ private Map<String, Map<Object, Object>> valueMapping;
private List<String> nullableKeys;
+ private List<String> notNullableKeys;
private List<String> nullValues;
- private List<String> sortKeys;
+
+ // Per-property update-merge strategies (OVERRIDE / APPEND / SUM / UNION / ...), keyed by target
+ // property name. When set, existing elements are merged instead of overwritten.
+ private Map<String, UpdateStrategy> updateStrategies;
// Time config
private String dateFormat;
+ private List<String> extraDateFormats;
private String timeZone;
+
+ // How raw string cells are parsed into SET/LIST elements.
+ private ListFormat listFormat;
+
+ // Label metadata (for schema creation)
+ private Long ttl;
+ private String ttlStartTime;
+ private String enableLabelIndex;
+ private Map<String, Object> userdata;
+
+ public enum LabelType {
+ VERTEX,
+ EDGE
+ }
+
+ @Data
+ public static class SourceTargetConfig implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+ private String label;
+ private List<String> idFields;
+ }
+
+ public Map<String, String> getFieldMapping() {
+ return fieldMapping == null ? Collections.emptyMap() : fieldMapping;
+ }
+
+ public Map<String, Map<Object, Object>> getValueMapping() {
+ return valueMapping == null ? Collections.emptyMap() : valueMapping;
+ }
+
+ public List<String> getNullValues() {
+ return nullValues == null ? Collections.emptyList() : nullValues;
+ }
+
+ public List<String> getNullableKeys() {
+ return nullableKeys == null ? Collections.emptyList() : nullableKeys;
+ }
+
+ public List<String> getNotNullableKeys() {
+ return notNullableKeys == null ? Collections.emptyList() : notNullableKeys;
+ }
+
+ public List<String> getSortKeys() {
+ return sortKeys == null ? Collections.emptyList() : sortKeys;
+ }
+
+ public List<String> getProperties() {
+ return properties == null ? Collections.emptyList() : properties;
+ }
+
+ public List<String> getIgnored() {
+ return ignored == null ? Collections.emptyList() : ignored;
+ }
+
+ public ListFormat getListFormat() {
+ return listFormat == null ? new ListFormat() : listFormat;
+ }
+
+ public List<String> getExtraDateFormats() {
+ return extraDateFormats == null ? Collections.emptyList() : extraDateFormats;
+ }
+
+ public Map<String, UpdateStrategy> getUpdateStrategies() {
+ return updateStrategies == null ? Collections.emptyMap() : updateStrategies;
+ }
+
+ public String getSourceTable() {
+ return sourceTable == null ? "" : sourceTable;
+ }
+
+ /**
+ * Whether this mapping is applicable to a writer serving the given table path. A mapping
+ * without {@code sourceTable} applies to every writer (backward compatible); a mapping with
+ * {@code sourceTable} only applies when the table path matches.
+ */
+ public boolean appliesTo(String tablePath) {
+ if (sourceTable == null || sourceTable.isEmpty()) {
+ return true;
+ }
+ return sourceTable.equals(tablePath);
+ }
+
+ /** Converts a legacy SchemaConfig to the new unified MappingConfig. */
+ public static MappingConfig fromLegacySchemaConfig(SchemaConfig schema) {
+ MappingConfig config = new MappingConfig();
+
+ // Element type & label
+ if (schema.getType() != null) {
+ config.setType(LabelType.valueOf(schema.getType().name()));
+ }
+ config.setLabel(schema.getLabel());
+
+ // Vertex config
+ config.setIdStrategy(schema.getIdStrategy());
+ config.setIdFields(schema.getIdFields());
+
+ // Edge config
+ if (schema.getSourceConfig() != null) {
+ SourceTargetConfig src = new SourceTargetConfig();
+ src.setLabel(schema.getSourceConfig().getLabel());
+ src.setIdFields(schema.getSourceConfig().getIdFields());
+ config.setSourceConfig(src);
+ }
+ if (schema.getTargetConfig() != null) {
+ SourceTargetConfig tgt = new SourceTargetConfig();
+ tgt.setLabel(schema.getTargetConfig().getLabel());
+ tgt.setIdFields(schema.getTargetConfig().getIdFields());
+ config.setTargetConfig(tgt);
+ }
+ config.setFrequency(schema.getFrequency());
+
+ // Properties
+ config.setProperties(schema.getProperties());
+
+ // Label metadata
+ config.setTtl(schema.getTtl());
+ config.setTtlStartTime(schema.getTtlStartTime());
+ config.setEnableLabelIndex(schema.getEnableLabelIndex());
+ config.setUserdata(schema.getUserdata());
+
+ // Flatten the nested mapping config
+ if (schema.getMapping() != null) {
+ MappingConfig legacy = schema.getMapping();
+ config.setFieldMapping(legacy.getFieldMapping());
+ config.setValueMapping(legacy.getValueMapping());
+ config.setNullableKeys(legacy.getNullableKeys());
+ config.setNullValues(legacy.getNullValues());
+ config.setSortKeys(legacy.getSortKeys());
+ config.setDateFormat(legacy.getDateFormat());
+ config.setTimeZone(legacy.getTimeZone());
+ }
+
+ return config;
+ }
}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/ReservedColumns.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/ReservedColumns.java
new file mode 100644
index 0000000000..17b1cd4cb4
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/ReservedColumns.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.hugegraph.config;
+
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Reserved output columns emitted by the HugeGraph Source. Every reserved column name starts with
+ * {@code ~}, which HugeGraph forbids for property key names, so they never collide with user
+ * properties.
+ *
+ * <p>These columns carry the pre-assembled HugeGraph element ids (the vertex id in {@code ~id}, the
+ * edge endpoint vertex ids in {@code ~source_id}/{@code ~target_id}). When a Sink mapping sets a
+ * single reserved column as its {@code idFields} / endpoint {@code idFields}, the mapper consumes
+ * that pre-assembled id directly instead of re-building one from primary-key columns — this is what
+ * makes a lossless HugeGraph → HugeGraph clone of edges and of CUSTOMIZE-id vertices possible.
+ */
+public final class ReservedColumns {
+
+ public static final String PREFIX = "~";
+
+ public static final String ID = "~id";
+ public static final String LABEL = "~label";
+ public static final String SOURCE_ID = "~source_id";
+ public static final String SOURCE_LABEL = "~source_label";
+ public static final String TARGET_ID = "~target_id";
+ public static final String TARGET_LABEL = "~target_label";
+
+ private ReservedColumns() {}
+
+ /** Whether {@code field} is a reserved Source column (starts with {@code ~}). */
+ public static boolean isReserved(String field) {
+ return field != null && field.startsWith(PREFIX);
+ }
+
+ /**
+ * Whether an {@code idFields} list requests raw-id passthrough: exactly one field, and that
+ * field is a reserved Source column carrying a pre-assembled id.
+ */
+ public static boolean isRawIdPassthrough(List<String> idFields) {
+ return idFields != null && idFields.size() == 1 && isReserved(idFields.get(0));
+ }
+
+ /**
+ * Removes reserved column names from the collection in-place, so callers that build a property
+ * set from all row fields (e.g. mappers and validators) can strip the non-property passthrough
+ * columns with one call. Returns {@code fields} for fluent use.
+ */
+ public static <T extends Collection<String>> T stripReserved(T fields) {
+ fields.removeIf(ReservedColumns::isReserved);
+ return fields;
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/SchemaConfig.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/SchemaConfig.java
index 4dfe3cddfe..abcfe399fc 100644
--- a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/SchemaConfig.java
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/config/SchemaConfig.java
@@ -26,9 +26,15 @@ import java.io.Serializable;
import java.util.List;
import java.util.Map;
+/**
+ * Legacy schema configuration object for backward compatibility. New configurations should use
+ * {@code mappings[]} with {@link MappingConfig} instead.
+ */
@Data
public class SchemaConfig implements Serializable {
+ private static final long serialVersionUID = 1L;
+
// General config
private LabelType type;
private String label;
@@ -52,7 +58,9 @@ public class SchemaConfig implements Serializable {
private SourceTargetConfig targetConfig;
private Frequency frequency;
- // Mapping Config
+ // Mapping Config (legacy nested object). Kept as MappingConfig to preserve the public
+ // getMapping()/setMapping() accessor descriptors of the previously released connector;
+ // MappingConfig already carries all the legacy nested fields.
private MappingConfig mapping;
public enum LabelType {
@@ -62,6 +70,7 @@ public class SchemaConfig implements Serializable {
@Data
public static class SourceTargetConfig implements Serializable {
+ private static final long serialVersionUID = 1L;
private String label;
private List<String> idFields;
}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/exception/HugeGraphConnectorErrorCode.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/exception/HugeGraphConnectorErrorCode.java
index e5608287c3..1c5205d394 100644
--- a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/exception/HugeGraphConnectorErrorCode.java
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/exception/HugeGraphConnectorErrorCode.java
@@ -27,6 +27,7 @@ public enum HugeGraphConnectorErrorCode implements SeaTunnelErrorCode {
BUFFER_ADD_FAILED("HUGEGRAPH-05", "BatchBuffer is already closed."),
INVALID_GRAPH_SCHEMA("HUGEGRAPH-06", "Invalid Graph Schema"),
ILLEGAL_CONFIG_ARGUMENT("HUGEGRAPH-07", "Illegal argument"),
+ SCHEMA_CREATION_FAILED("HUGEGRAPH-08", "Schema auto-creation failed"),
;
private final String code;
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/mapper/EdgeMapper.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/mapper/EdgeMapper.java
index 2e22a76fa8..ac37738d85 100644
--- a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/mapper/EdgeMapper.java
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/mapper/EdgeMapper.java
@@ -20,12 +20,15 @@ package org.apache.seatunnel.connectors.seatunnel.hugegraph.mapper;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.client.HugeGraphClient;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.MappingConfig;
-import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.SchemaConfig;
-import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.SchemaConfig.SourceTargetConfig;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.MappingConfig.SourceTargetConfig;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.ReservedColumns;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorErrorCode;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorException;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.utils.DataTypeUtil;
+import org.apache.hugegraph.serializer.direct.util.SplicingIdGenerator;
+import org.apache.hugegraph.structure.GraphElement;
+import org.apache.hugegraph.structure.constant.Frequency;
import org.apache.hugegraph.structure.constant.IdStrategy;
import org.apache.hugegraph.structure.graph.Edge;
import org.apache.hugegraph.structure.schema.PropertyKey;
@@ -38,107 +41,266 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
-import java.util.stream.Collectors;
public class EdgeMapper implements GraphDataMapper {
- private final SchemaConfig schemaConfig;
private final MappingConfig mappingConfig;
private final Map<String, Integer> fieldsIndex;
private final HugeGraphClient client;
private final String labelId;
private final Map<String, PropertyKey> propertyKeyCache;
+ private final Set<String> propertySourceFields;
+ private final Set<String> edgeIdSourceFields;
+
+ // Cached at construction time to avoid per-row schema queries
+ private final String sourceVertexLabelId;
+ private final IdStrategy sourceIdStrategy;
+ private final String targetVertexLabelId;
+ private final IdStrategy targetIdStrategy;
+ private final boolean unfoldSource;
+ private final boolean unfoldTarget;
public EdgeMapper(
- SchemaConfig schemaConfig, Map<String, Integer> fieldsIndex, HugeGraphClient client) {
- this.schemaConfig = schemaConfig;
- this.mappingConfig = getMappingConfig();
+ MappingConfig mappingConfig, Map<String, Integer> fieldsIndex, HugeGraphClient client) {
+ this.mappingConfig = mappingConfig;
this.client = client;
- this.labelId = client.getEdgeLabelId(schemaConfig.getLabel());
+ this.labelId = client.getEdgeLabelId(mappingConfig.getLabel());
this.fieldsIndex = fieldsIndex;
- this.propertyKeyCache = getPropertyKeyCache();
+ this.edgeIdSourceFields = resolveEdgeIdSourceFields();
+ this.propertySourceFields = resolvePropertySourceFields();
+ this.propertyKeyCache = buildPropertyKeyCache();
+ this.unfoldSource = mappingConfig.isUnfoldSource();
+ this.unfoldTarget = mappingConfig.isUnfoldTarget();
+
+ // Cache source/target vertex metadata to avoid per-row schema queries
+ this.sourceVertexLabelId =
+ client.getVertexLabelId(mappingConfig.getSourceConfig().getLabel());
+ this.sourceIdStrategy = client.getIdStrategy(mappingConfig.getSourceConfig().getLabel());
+ this.targetVertexLabelId =
+ client.getVertexLabelId(mappingConfig.getTargetConfig().getLabel());
+ this.targetIdStrategy = client.getIdStrategy(mappingConfig.getTargetConfig().getLabel());
+ }
+
+ @Override
+ public boolean isUnfoldEnabled() {
+ return unfoldSource || unfoldTarget;
+ }
+
+ private Set<String> resolveEdgeIdSourceFields() {
+ Set<String> fields = new HashSet<>();
+ if (mappingConfig.getSourceConfig() != null
+ && mappingConfig.getSourceConfig().getIdFields() != null) {
+ fields.addAll(mappingConfig.getSourceConfig().getIdFields());
+ }
+ if (mappingConfig.getTargetConfig() != null
+ && mappingConfig.getTargetConfig().getIdFields() != null) {
+ fields.addAll(mappingConfig.getTargetConfig().getIdFields());
+ }
+ return fields;
}
- private MappingConfig getMappingConfig() {
- MappingConfig mapping =
- schemaConfig.getMapping() == null ? new MappingConfig() : schemaConfig.getMapping();
- if (mapping.getFieldMapping() == null) {
- mapping.setFieldMapping(Collections.emptyMap());
+ private Set<String> resolvePropertySourceFields() {
+ Set<String> fields = new HashSet<>();
+ if (mappingConfig.getProperties().isEmpty()) {
+ // Implicit mode ("write every row field as a property") — endpoint id fields locate
+ // vertices and would otherwise be duplicated onto the edge; drop them here.
+ fields.addAll(fieldsIndex.keySet());
+ fields.removeAll(edgeIdSourceFields);
+ fields.removeAll(reservedSourceFields(fieldsIndex.keySet()));
+ // `ignored` blacklist only applies in implicit mode.
+ fields.removeAll(mappingConfig.getIgnored());
+ } else {
+ // Explicit mode — respect the user's list verbatim. If they list an endpoint field it
+ // is genuinely meant to appear as an edge property, matching what SchemaManager
+ // creates on the server.
+ fields.addAll(mappingConfig.getProperties());
}
- if (mapping.getValueMapping() == null) {
- mapping.setValueMapping(Collections.emptyMap());
+ // Sort keys are always edge properties (server-side EdgeId requires them).
+ fields.addAll(mappingConfig.getSortKeys());
+ return fields;
+ }
+
+ /**
+ * Reserved fields emitted by the HugeGraph Source (e.g. {@code ~id}, {@code ~label}). They are
+ * not valid HugeGraph property key names — including them would fail at server-side property
+ * key creation — so drop them from an implicit round-trip.
+ */
+ private static Set<String> reservedSourceFields(Set<String> allFields) {
+ Set<String> reserved = new HashSet<>();
+ for (String field : allFields) {
+ if (field != null && field.startsWith("~")) {
+ reserved.add(field);
+ }
}
- schemaConfig.setMapping(mapping);
- return mapping;
+ return reserved;
}
- private HashMap<String, PropertyKey> getPropertyKeyCache() {
+ private HashMap<String, PropertyKey> buildPropertyKeyCache() {
HashMap<String, PropertyKey> cache = new HashMap<>();
- Map<String, String> fieldMapping = mappingConfig.getFieldMapping();
- for (String fieldName : fieldsIndex.keySet()) {
- String propertyName = fieldMapping.getOrDefault(fieldName, fieldName);
- cache.put(propertyName, client.getPropertyKey(propertyName));
+ Map<String, String> fm = mappingConfig.getFieldMapping();
+
+ // Cache for property fields
+ for (String sourceField : propertySourceFields) {
+ String propName = fm.getOrDefault(sourceField, sourceField);
+ if (!cache.containsKey(propName)) {
+ cache.put(propName, client.getPropertyKey(propName));
+ }
+ }
+
+ // Cache for id fields (needed for type conversion during ID construction)
+ for (String idField : edgeIdSourceFields) {
+ String propName = fm.getOrDefault(idField, idField);
+ if (!cache.containsKey(propName)) {
+ PropertyKey pk = client.getPropertyKeyOrNull(propName);
+ if (pk != null) {
+ cache.put(propName, pk);
+ }
+ }
}
+
return cache;
}
@Override
public Edge map(SeaTunnelRow row) {
- // 1. Build source and target vertex IDs
- Object sourceId = buildVertexId(row, schemaConfig.getSourceConfig());
- Object targetId = buildVertexId(row, schemaConfig.getTargetConfig());
+ Object sourceId = buildVertexId(row, mappingConfig.getSourceConfig());
+ Object targetId = buildVertexId(row, mappingConfig.getTargetConfig());
- // If source or target ID can't be built, we can't create the edge
if (sourceId == null || targetId == null) {
return null;
}
+ return buildEdge(row, sourceId, targetId);
+ }
+
+ /**
+ * INSERT/append-path expansion: when {@code unfold_source} / {@code unfold_target} is set, a
+ * list-valued endpoint id cell expands into multiple endpoint ids and edges are produced for
+ * the cartesian product. Only CUSTOMIZE endpoints are supported (validated in SchemaValidator).
+ */
+ @Override
+ public List<GraphElement> mapAll(SeaTunnelRow row) {
+ if (!unfoldSource && !unfoldTarget) {
+ Edge edge = map(row);
+ return edge == null ? Collections.emptyList() : Collections.singletonList(edge);
+ }
+ List<Object> sourceIds =
+ buildVertexIdList(row, mappingConfig.getSourceConfig(), unfoldSource);
+ List<Object> targetIds =
+ buildVertexIdList(row, mappingConfig.getTargetConfig(), unfoldTarget);
+ if (sourceIds.isEmpty() || targetIds.isEmpty()) {
+ return Collections.emptyList();
+ }
+ List<GraphElement> result = new ArrayList<>(sourceIds.size() * targetIds.size());
+ for (Object sourceId : sourceIds) {
+ for (Object targetId : targetIds) {
+ result.add(buildEdge(row, sourceId, targetId));
+ }
+ }
+ return result;
+ }
+
+ private List<Object> buildVertexIdList(
+ SeaTunnelRow row, SourceTargetConfig config, boolean unfoldEndpoint) {
+ if (!unfoldEndpoint) {
+ Object id = buildVertexId(row, config);
+ return id == null ? Collections.emptyList() : Collections.singletonList(id);
+ }
+ boolean isSource = config == mappingConfig.getSourceConfig();
+ IdStrategy strategy = isSource ? sourceIdStrategy : targetIdStrategy;
+ String idField = config.getIdFields().get(0);
+ Integer idx = fieldsIndex.get(idField);
+ if (idx == null) {
+ return Collections.emptyList();
+ }
+ Object raw = row.getField(idx);
+ if (isConsideredNull(raw)) {
+ return Collections.emptyList();
+ }
+ List<Object> elements = DataTypeUtil.splitField(idField, raw);
+ List<Object> ids = new ArrayList<>(elements.size());
+ for (Object elem : elements) {
+ if (isConsideredNull(elem)) {
+ continue;
+ }
+ ids.add(coerceCustomizeId(strategy, elem));
+ }
+ return ids;
+ }
+
+ private static Object coerceCustomizeId(IdStrategy strategy, Object elem) {
+ switch (strategy) {
+ case CUSTOMIZE_STRING:
+ return VertexMapper.checkVertexIdLength(String.valueOf(elem));
+ case CUSTOMIZE_NUMBER:
+ return VertexMapper.coerceNumberId(elem);
+ case CUSTOMIZE_UUID:
+ return elem instanceof UUID ? elem : UUID.fromString(String.valueOf(elem));
+ default:
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ "unfold requires a CUSTOMIZE_STRING/NUMBER/UUID endpoint id strategy, but got "
+ + strategy);
+ }
+ }
- // 2. Create edge and set identifiers
- Edge edge = new Edge(schemaConfig.getLabel());
+ private Edge buildEdge(SeaTunnelRow row, Object sourceId, Object targetId) {
+ Edge edge = new Edge(mappingConfig.getLabel());
edge.sourceId(sourceId);
edge.targetId(targetId);
- edge.sourceLabel(schemaConfig.getSourceConfig().getLabel());
- edge.targetLabel(schemaConfig.getTargetConfig().getLabel());
-
- // 3. Set properties
- Set<String> idFields = new HashSet<>();
- idFields.addAll(schemaConfig.getSourceConfig().getIdFields());
- idFields.addAll(schemaConfig.getTargetConfig().getIdFields());
+ edge.sourceLabel(mappingConfig.getSourceConfig().getLabel());
+ edge.targetLabel(mappingConfig.getTargetConfig().getLabel());
- Map<String, String> fieldMapping = new HashMap<>(mappingConfig.getFieldMapping());
+ Map<String, String> fm = mappingConfig.getFieldMapping();
+ for (String sourceField : propertySourceFields) {
+ Integer index = fieldsIndex.get(sourceField);
+ if (index == null) {
+ continue;
+ }
- for (Map.Entry<String, Integer> fieldEntry : fieldsIndex.entrySet()) {
- String fieldName = fieldEntry.getKey();
- String propertyName = fieldMapping.getOrDefault(fieldName, fieldName);
- Object rawValue = row.getField(fieldEntry.getValue());
- PropertyKey propertyKey = propertyKeyCache.get(propertyName);
+ String propName = fm.getOrDefault(sourceField, sourceField);
+ Object rawValue = row.getField(index);
+ PropertyKey propertyKey = propertyKeyCache.get(propName);
- // Skip fields used for source/target vertex IDs
- if (idFields.contains(fieldName) || isConsideredNull(rawValue)) {
+ if (isConsideredNull(rawValue)) {
continue;
}
- Object fieldValue =
+ Object converted =
DataTypeUtil.convert(
rawValue,
propertyKey,
mappingConfig.getDateFormat(),
- mappingConfig.getTimeZone());
-
- edge.property(propertyName, getMappedValue(fieldValue));
+ mappingConfig.getTimeZone(),
+ mappingConfig.getExtraDateFormats(),
+ mappingConfig.getListFormat());
+ edge.property(propName, getMappedValue(sourceField, converted));
}
return edge;
}
private Object buildVertexId(SeaTunnelRow row, SourceTargetConfig config) {
+ boolean isSource = config == mappingConfig.getSourceConfig();
+ String vertexLabelId = isSource ? sourceVertexLabelId : targetVertexLabelId;
+ IdStrategy strategy = isSource ? sourceIdStrategy : targetIdStrategy;
+ List<String> idFields = config.getIdFields();
+
+ // Raw-id passthrough: the endpoint id is already assembled in a reserved Source column
+ // (~source_id / ~target_id). Use it directly so an edge can be cloned without re-deriving
+ // the endpoint vertex ids from primary-key columns. Works for any endpoint id strategy
+ // (including AUTOMATIC) because we only reuse the id string, never rebuild the vertex.
+ if (ReservedColumns.isRawIdPassthrough(idFields)) {
+ Integer idx = fieldsIndex.get(idFields.get(0));
+ Object raw = idx == null ? null : row.getField(idx);
+ if (isConsideredNull(raw)) {
+ return null;
+ }
+ return coerceRawVertexId(String.valueOf(raw), strategy);
+ }
- String vertexLabelId = client.getVertexLabelId(config.getLabel());
- IdStrategy strategy = client.getIdStrategy(config.getLabel());
if (strategy == null || strategy == IdStrategy.AUTOMATIC) {
return null;
}
- List<String> idFields = config.getIdFields();
switch (strategy) {
case PRIMARY_KEY:
List<Object> pkValues = getFieldValues(row, idFields);
@@ -153,7 +315,7 @@ public class EdgeMapper implements GraphDataMapper {
|| stringValues.stream().anyMatch(this::isConsideredNull)) {
return null;
}
- return stringValues.stream().map(String::valueOf).collect(Collectors.joining(":"));
+ return VertexMapper.spliceCustomizeStringId(stringValues);
case CUSTOMIZE_NUMBER:
List<Object> numberValues = getFieldValues(row, idFields);
if (numberValues.size() != 1) {
@@ -163,11 +325,7 @@ public class EdgeMapper implements GraphDataMapper {
if (isConsideredNull(numValue)) {
return null;
}
- if (numValue instanceof Number) {
- return ((Number) numValue).longValue();
- } else {
- return Long.parseLong(String.valueOf(numValue));
- }
+ return VertexMapper.coerceNumberId(numValue);
case CUSTOMIZE_UUID:
List<Object> uuidValues = getFieldValues(row, idFields);
if (uuidValues.size() != 1) {
@@ -187,16 +345,16 @@ public class EdgeMapper implements GraphDataMapper {
private List<Object> getFieldValues(SeaTunnelRow row, List<String> fields) {
List<Object> values = new ArrayList<>(fields.size());
- Map<String, String> fieldMapping = mappingConfig.getFieldMapping();
+ Map<String, String> fm = mappingConfig.getFieldMapping();
for (String fieldName : fields) {
-
Integer index = fieldsIndex.get(fieldName);
if (index == null) {
throw new HugeGraphConnectorException(
HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
String.format(
- "Field '%s' specified in id_fields not found in row schema. Available fields: %s",
- fieldName, fieldsIndex.keySet()));
+ "Mapping[EDGE/%s]: Field '%s' specified in idFields not found in row schema. "
+ + "Available fields: %s",
+ mappingConfig.getLabel(), fieldName, fieldsIndex.keySet()));
}
Object rawValue = row.getField(index);
@@ -204,60 +362,136 @@ public class EdgeMapper implements GraphDataMapper {
continue;
}
- String propertyName = fieldMapping.getOrDefault(fieldName, fieldName);
- PropertyKey propertyKey = propertyKeyCache.get(propertyName);
+ String propName = fm.getOrDefault(fieldName, fieldName);
+ PropertyKey propertyKey = propertyKeyCache.get(propName);
+
+ if (propertyKey != null) {
+ Object converted =
+ DataTypeUtil.convert(
+ rawValue,
+ propertyKey,
+ mappingConfig.getDateFormat(),
+ mappingConfig.getTimeZone(),
+ mappingConfig.getExtraDateFormats(),
+ mappingConfig.getListFormat());
+ values.add(getMappedValue(fieldName, converted));
+ } else {
+ values.add(getMappedValue(fieldName, rawValue));
+ }
+ }
+ return values;
+ }
- Object fieldValue =
- DataTypeUtil.convert(
- rawValue,
- propertyKey,
- mappingConfig.getDateFormat(),
- mappingConfig.getTimeZone());
+ /**
+ * Extracts the Edge ID matching the HugeGraph server-side 5-part EdgeId format:
+ * {ownerVertexId}>{edgeLabelId}>{subEdgeLabelId}>{sortValues}>{otherVertexId}
+ *
+ * <p>For general (non-hierarchical) edge labels the sub-label ID equals the edge label ID, so
+ * the label ID appears twice. SINGLE frequency edges have an empty sortValues segment; MULTIPLE
+ * frequency uses the sortKeys values. Vertex IDs are prefixed with 'S' for String IDs, 'L' for
+ * Number IDs, and 'U' for UUID IDs.
+ */
+ @Override
+ public Object extractId(SeaTunnelRow row) {
+ Object sourceId = buildVertexId(row, mappingConfig.getSourceConfig());
+ Object targetId = buildVertexId(row, mappingConfig.getTargetConfig());
- values.add(getMappedValue(fieldValue));
+ if (sourceId == null || targetId == null) {
+ return null;
}
- return values;
+
+ return spliceEdgeId(sourceId, targetId, labelId, getSortKeyValues(row));
}
- private boolean isConsideredNull(Object value) {
- if (value == null) {
- return true;
+ /**
+ * Splices the HugeGraph server-side 5-part EdgeId. Package-private and static so the
+ * format-sensitive layout (vertex-id prefix, doubled label id for the sub-label segment,
+ * sortValues segment) is unit-testable without a live server — this is the DELETE-correctness
+ * path.
+ *
+ * <p>Uses HugeGraph's own {@link SplicingIdGenerator} so the encoding matches the server:
+ * {@code concat} joins the five segments with {@code '>'} and backtick-escapes any {@code '>'}
+ * inside a segment; {@code concatValues} joins the sort-key values with {@code '!'} and
+ * backtick-escapes any {@code '!'}. Vertex ids carry the type prefix HugeGraph expects: {@code
+ * 'L'} for numbers, {@code 'U'} for UUIDs, {@code 'S'} for strings.
+ */
+ static String spliceEdgeId(
+ Object sourceId, Object targetId, String labelId, List<Object> sortValues) {
+ String sort =
+ (sortValues == null || sortValues.isEmpty())
+ ? ""
+ : SplicingIdGenerator.concatValues(sortValues);
+ return SplicingIdGenerator.concat(
+ vertexIdString(sourceId), labelId, labelId, sort, vertexIdString(targetId));
+ }
+
+ /**
+ * Converts a reserved-column id string ({@code ~source_id}/{@code ~target_id}) back into the
+ * Java id type the endpoint vertex uses, so {@link #vertexIdString} re-applies the correct
+ * {@code L}/{@code U}/{@code S} prefix. PRIMARY_KEY / CUSTOMIZE_STRING ids stay strings (e.g.
+ * {@code "1:marko"} → {@code "S1:marko"}); CUSTOMIZE_NUMBER / AUTOMATIC parse to a long;
+ * CUSTOMIZE_UUID parses to a UUID.
+ */
+ private static Object coerceRawVertexId(String raw, IdStrategy strategy) {
+ if (strategy == IdStrategy.CUSTOMIZE_NUMBER || strategy == IdStrategy.AUTOMATIC) {
+ return Long.parseLong(raw);
}
- List<String> nullValues = mappingConfig.getNullValues();
- if (nullValues == null || nullValues.isEmpty()) {
- return false;
+ if (strategy == IdStrategy.CUSTOMIZE_UUID) {
+ return UUID.fromString(raw);
}
- return nullValues.contains(String.valueOf(value));
+ return raw;
}
- private Object getMappedValue(Object originalValue) {
- Map<Object, Object> valueMapping = mappingConfig.getValueMapping();
- if (valueMapping.isEmpty()) {
- return originalValue;
+ /** Prepends the HugeGraph vertex-id type prefix: 'L' number, 'U' UUID, 'S' string. */
+ private static String vertexIdString(Object id) {
+ String prefix;
+ if (id instanceof Number) {
+ prefix = "L";
+ } else if (id instanceof UUID) {
+ prefix = "U";
+ } else {
+ prefix = "S";
}
- return valueMapping.getOrDefault(originalValue, originalValue);
+ return prefix + id;
}
- private String spliceVertexId(String vertexLabelId, List<Object> primaryValues) {
- String joinedValues =
- primaryValues.stream().map(Object::toString).collect(Collectors.joining("!"));
- return String.format("%s:%s", vertexLabelId, joinedValues);
+ private List<Object> getSortKeyValues(SeaTunnelRow row) {
+ Frequency frequency = mappingConfig.getFrequency();
+ if (frequency == null || frequency == Frequency.SINGLE) {
+ return Collections.emptyList();
+ }
+ List<String> sortKeys = mappingConfig.getSortKeys();
+ if (sortKeys.isEmpty()) {
+ return Collections.emptyList();
+ }
+ return getFieldValues(row, sortKeys);
}
- private String getSortedKeyValues(SeaTunnelRow row) {
- List<String> sortedKeys = mappingConfig.getSortKeys();
- if (sortedKeys == null || sortedKeys.isEmpty()) {
- return String.valueOf(labelId);
+ private boolean isConsideredNull(Object value) {
+ if (value == null) {
+ return true;
}
- List<Object> skValues = getFieldValues(row, sortedKeys);
- return skValues.stream().map(Object::toString).collect(Collectors.joining(","));
+ List<String> nullValues = mappingConfig.getNullValues();
+ return !nullValues.isEmpty() && nullValues.contains(String.valueOf(value));
}
- @Override
- public Object extractId(SeaTunnelRow row) {
- Object sourceId = buildVertexId(row, schemaConfig.getSourceConfig());
- Object targetId = buildVertexId(row, schemaConfig.getTargetConfig());
- String sortedKeyValues = getSortedKeyValues(row);
- return String.format("S%s>%s>%s>>S%s", sourceId, labelId, sortedKeyValues, targetId);
+ private Object getMappedValue(String sourceField, Object originalValue) {
+ Map<String, Map<Object, Object>> vm = mappingConfig.getValueMapping();
+ if (vm.isEmpty()) {
+ return originalValue;
+ }
+ Map<Object, Object> perField = vm.get(sourceField);
+ if (perField == null || perField.isEmpty()) {
+ return originalValue;
+ }
+ return perField.getOrDefault(originalValue, originalValue);
+ }
+
+ private String spliceVertexId(String vertexLabelId, List<Object> primaryValues) {
+ // HugeGraph primary-key vertex id = {vertexLabelId}:{concatValues(pk)}; concatValues joins
+ // with '!' and backtick-escapes any '!' so pk values containing the separator still match.
+ return VertexMapper.checkVertexIdLength(
+ String.format(
+ "%s:%s", vertexLabelId, SplicingIdGenerator.concatValues(primaryValues)));
}
}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/mapper/GraphDataMapper.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/mapper/GraphDataMapper.java
index b5a669625b..a2fb8a1d1b 100644
--- a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/mapper/GraphDataMapper.java
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/mapper/GraphDataMapper.java
@@ -22,22 +22,36 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.hugegraph.structure.GraphElement;
import java.io.Serializable;
+import java.util.Collections;
+import java.util.List;
public interface GraphDataMapper extends Serializable {
/**
- * Maps a SeaTunnelRow to a HugeGraph GraphElement (Vertex or Edge).
- *
- * @param row The input SeaTunnelRow.
- * @return The resulting GraphElement.
+ * Maps a SeaTunnelRow to a HugeGraph GraphElement (Vertex or Edge). Returns null if the element
+ * should be skipped (e.g. null ID fields matched by nullValues).
*/
GraphElement map(SeaTunnelRow row);
/**
- * Extracts the ID from a SeaTunnelRow.
- *
- * @param row The input SeaTunnelRow.
- * @return The extracted ID object.
+ * Maps a row to one or more graph elements. Without unfold this is just {@link #map} wrapped in
+ * a list; with unfold enabled a list-valued id cell expands into multiple elements. Used on the
+ * INSERT/append path only.
+ */
+ default List<GraphElement> mapAll(SeaTunnelRow row) {
+ GraphElement element = map(row);
+ return element == null ? Collections.emptyList() : Collections.singletonList(element);
+ }
+
+ /** Whether this mapper expands one row into multiple elements (unfold). */
+ default boolean isUnfoldEnabled() {
+ return false;
+ }
+
+ /**
+ * Extracts the graph element ID from a SeaTunnelRow. The ID format must match the server-side
+ * format to ensure DELETE operations target the correct element. Returns null if the ID cannot
+ * be built (e.g. null ID fields).
*/
Object extractId(SeaTunnelRow row);
}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/mapper/VertexMapper.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/mapper/VertexMapper.java
index 9f11310583..830189e307 100644
--- a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/mapper/VertexMapper.java
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/mapper/VertexMapper.java
@@ -20,122 +20,242 @@ package org.apache.seatunnel.connectors.seatunnel.hugegraph.mapper;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.client.HugeGraphClient;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.MappingConfig;
-import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.SchemaConfig;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.ReservedColumns;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorErrorCode;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorException;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.utils.DataTypeUtil;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.utils.E;
+import org.apache.hugegraph.serializer.direct.util.SplicingIdGenerator;
+import org.apache.hugegraph.structure.GraphElement;
import org.apache.hugegraph.structure.constant.IdStrategy;
import org.apache.hugegraph.structure.graph.Vertex;
import org.apache.hugegraph.structure.schema.PropertyKey;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
public class VertexMapper implements GraphDataMapper {
- private final SchemaConfig schemaConfig;
private final MappingConfig mappingConfig;
private final Map<String, Integer> fieldsIndex;
private final String labelId;
private final HugeGraphClient client;
private final Map<String, PropertyKey> propertyKeyCache;
+ private final Set<String> propertySourceFields;
+ private final boolean unfold;
public VertexMapper(
- SchemaConfig schemaConfig, Map<String, Integer> fieldsIndex, HugeGraphClient client) {
- this.schemaConfig = schemaConfig;
- this.mappingConfig = getMappingConfig();
+ MappingConfig mappingConfig, Map<String, Integer> fieldsIndex, HugeGraphClient client) {
+ this.mappingConfig = mappingConfig;
this.client = client;
- this.labelId = client.getVertexLabelId(schemaConfig.getLabel());
+ this.labelId = client.getVertexLabelId(mappingConfig.getLabel());
this.fieldsIndex = fieldsIndex;
- this.propertyKeyCache = getPropertyKeyCache();
+ this.propertySourceFields = resolvePropertySourceFields();
+ this.propertyKeyCache = buildPropertyKeyCache();
+ this.unfold = mappingConfig.isUnfold();
}
- private MappingConfig getMappingConfig() {
- MappingConfig mapping =
- schemaConfig.getMapping() == null ? new MappingConfig() : schemaConfig.getMapping();
- if (mapping.getFieldMapping() == null) {
- mapping.setFieldMapping(Collections.emptyMap());
+ @Override
+ public boolean isUnfoldEnabled() {
+ return unfold;
+ }
+
+ private Set<String> resolvePropertySourceFields() {
+ Set<String> fields = new HashSet<>();
+ if (mappingConfig.getProperties().isEmpty()) {
+ fields.addAll(fieldsIndex.keySet());
+ // Drop reserved columns emitted by HugeGraph Source (~id, ~label, ...) — they are not
+ // valid HugeGraph property key names, so an implicit Source→Sink round-trip would
+ // otherwise attempt to create them on the server.
+ ReservedColumns.stripReserved(fields);
+ // `ignored` blacklist only applies in implicit mode (an explicit `properties`
+ // whitelist already lists exactly what to keep).
+ fields.removeAll(mappingConfig.getIgnored());
+ } else {
+ fields.addAll(mappingConfig.getProperties());
}
- if (mapping.getValueMapping() == null) {
- mapping.setValueMapping(Collections.emptyMap());
+ // PRIMARY_KEY idFields are always written as properties.
+ if (mappingConfig.getIdStrategy() == IdStrategy.PRIMARY_KEY
+ && mappingConfig.getIdFields() != null) {
+ fields.addAll(mappingConfig.getIdFields());
}
- schemaConfig.setMapping(mapping);
- return mapping;
+ return fields;
}
- private HashMap<String, PropertyKey> getPropertyKeyCache() {
+ private HashMap<String, PropertyKey> buildPropertyKeyCache() {
HashMap<String, PropertyKey> cache = new HashMap<>();
- Map<String, String> fieldMapping = mappingConfig.getFieldMapping();
- for (String fieldName : fieldsIndex.keySet()) {
- String propertyName = fieldMapping.getOrDefault(fieldName, fieldName);
- cache.put(propertyName, client.getPropertyKey(propertyName));
+ Map<String, String> fm = mappingConfig.getFieldMapping();
+ for (String sourceField : propertySourceFields) {
+ String propName = fm.getOrDefault(sourceField, sourceField);
+ if (!cache.containsKey(propName)) {
+ cache.put(propName, client.getPropertyKey(propName));
+ }
+ }
+ if (mappingConfig.getIdFields() != null) {
+ for (String idField : mappingConfig.getIdFields()) {
+ String propName = fm.getOrDefault(idField, idField);
+ if (!cache.containsKey(propName)) {
+ PropertyKey propertyKey = client.getPropertyKeyOrNull(propName);
+ if (propertyKey != null) {
+ cache.put(propName, propertyKey);
+ }
+ }
+ }
}
return cache;
}
@Override
public Vertex map(SeaTunnelRow row) {
- String label = schemaConfig.getLabel();
+ String label = mappingConfig.getLabel();
E.checkArgument(label != null && !label.isEmpty(), "Vertex label can't be null or empty.");
Vertex vertex = new Vertex(label);
- // 1. Set vertex ID
Object id = extractId(row);
- if (id == null && schemaConfig.getIdStrategy() != IdStrategy.AUTOMATIC) {
+ if (id == null && mappingConfig.getIdStrategy() != IdStrategy.AUTOMATIC) {
return null;
}
- if (id != null && schemaConfig.getIdStrategy() != IdStrategy.PRIMARY_KEY) {
+ if (id != null && mappingConfig.getIdStrategy() != IdStrategy.PRIMARY_KEY) {
vertex.id(id);
}
- // 2. Set properties
- Map<String, String> fieldMapping = mappingConfig.getFieldMapping();
+ applyProperties(vertex, row, null);
+ return vertex;
+ }
- for (Map.Entry<String, Integer> fieldEntry : fieldsIndex.entrySet()) {
+ /**
+ * INSERT/append-path expansion: when {@code unfold} is set, a list-valued CUSTOMIZE id cell
+ * produces one vertex per element (all sharing the same non-id properties). Without unfold this
+ * is just {@link #map} wrapped in a list.
+ */
+ @Override
+ public List<GraphElement> mapAll(SeaTunnelRow row) {
+ if (!unfold) {
+ Vertex vertex = map(row);
+ return vertex == null ? Collections.emptyList() : Collections.singletonList(vertex);
+ }
+ IdStrategy strategy = mappingConfig.getIdStrategy();
+ String idField = mappingConfig.getIdFields().get(0);
+ Integer idx = fieldsIndex.get(idField);
+ if (idx == null) {
+ return Collections.emptyList();
+ }
+ Object raw = row.getField(idx);
+ if (isConsideredNull(raw)) {
+ return Collections.emptyList();
+ }
+ List<Object> elements = DataTypeUtil.splitField(idField, raw);
+ List<GraphElement> result = new ArrayList<>(elements.size());
+ for (Object elem : elements) {
+ if (isConsideredNull(elem)) {
+ continue;
+ }
+ Vertex vertex = new Vertex(mappingConfig.getLabel());
+ vertex.id(coerceCustomizeId(strategy, elem));
+ // The id field is the unfolded source, not a property — skip it.
+ applyProperties(vertex, row, idField);
+ result.add(vertex);
+ }
+ return result;
+ }
- String fieldName = fieldEntry.getKey();
- String propertyName = fieldMapping.getOrDefault(fieldName, fieldName);
- Object rawValue = row.getField(fieldEntry.getValue());
- PropertyKey propertyKey = propertyKeyCache.get(propertyName);
+ private static Object coerceCustomizeId(IdStrategy strategy, Object elem) {
+ switch (strategy) {
+ case CUSTOMIZE_STRING:
+ return checkVertexIdLength(String.valueOf(elem));
+ case CUSTOMIZE_NUMBER:
+ return coerceNumberId(elem);
+ case CUSTOMIZE_UUID:
+ return elem instanceof UUID ? elem : UUID.fromString(String.valueOf(elem));
+ default:
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ "unfold requires a CUSTOMIZE_STRING/NUMBER/UUID id strategy, but got "
+ + strategy);
+ }
+ }
+
+ private void applyProperties(Vertex vertex, SeaTunnelRow row, String skipField) {
+ Map<String, String> fm = mappingConfig.getFieldMapping();
+ for (String sourceField : propertySourceFields) {
+ if (sourceField.equals(skipField)) {
+ continue;
+ }
+ Integer index = fieldsIndex.get(sourceField);
+ if (index == null) {
+ continue;
+ }
+
+ String propName = fm.getOrDefault(sourceField, sourceField);
+ Object rawValue = row.getField(index);
+ PropertyKey propertyKey = propertyKeyCache.get(propName);
if (isConsideredNull(rawValue)) {
continue;
}
- Object fieldValue =
+ Object converted =
DataTypeUtil.convert(
rawValue,
propertyKey,
mappingConfig.getDateFormat(),
- mappingConfig.getTimeZone());
-
- vertex.property(propertyName, getMappedValue(fieldValue));
+ mappingConfig.getTimeZone(),
+ mappingConfig.getExtraDateFormats(),
+ mappingConfig.getListFormat());
+ vertex.property(propName, getMappedValue(sourceField, converted));
}
-
- return vertex;
}
@Override
public Object extractId(SeaTunnelRow row) {
- IdStrategy strategy = schemaConfig.getIdStrategy();
+ IdStrategy strategy = mappingConfig.getIdStrategy();
if (strategy == null || strategy == IdStrategy.AUTOMATIC) {
return null;
}
- List<String> idFields = schemaConfig.getIdFields();
+ List<String> idFields = mappingConfig.getIdFields();
E.checkArgument(
idFields != null && !idFields.isEmpty(),
"The 'idFields' must be specified for ID strategy '%s'.",
strategy);
+ // Raw-id passthrough: the vertex id is already assembled in the reserved ~id Source column.
+ // Only meaningful for CUSTOMIZE_* strategies (the ones that accept an externally supplied
+ // id); PRIMARY_KEY derives its id from property values and AUTOMATIC is server-assigned, so
+ // both are rejected at config time in SchemaValidator.
+ if (ReservedColumns.isRawIdPassthrough(idFields)) {
+ Integer idx = fieldsIndex.get(idFields.get(0));
+ Object raw = idx == null ? null : row.getField(idx);
+ if (isConsideredNull(raw)) {
+ return null;
+ }
+ switch (strategy) {
+ case CUSTOMIZE_STRING:
+ return checkVertexIdLength(String.valueOf(raw));
+ case CUSTOMIZE_NUMBER:
+ return coerceNumberId(raw);
+ case CUSTOMIZE_UUID:
+ return UUID.fromString(String.valueOf(raw));
+ default:
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "Mapping[VERTEX/%s]: idFields '%s' (raw-id passthrough) requires a "
+ + "CUSTOMIZE_STRING/NUMBER/UUID id strategy, but got '%s'.",
+ mappingConfig.getLabel(), idFields.get(0), strategy));
+ }
+ }
+
switch (strategy) {
case PRIMARY_KEY:
List<Object> pkValues = getFieldValues(row, idFields);
@@ -150,7 +270,7 @@ public class VertexMapper implements GraphDataMapper {
|| stringValues.stream().anyMatch(this::isConsideredNull)) {
return null;
}
- return stringValues.stream().map(String::valueOf).collect(Collectors.joining(":"));
+ return spliceCustomizeStringId(stringValues);
case CUSTOMIZE_NUMBER:
List<Object> numberValues = getFieldValues(row, idFields);
if (numberValues.size() != 1) {
@@ -160,11 +280,7 @@ public class VertexMapper implements GraphDataMapper {
if (isConsideredNull(numValue)) {
return null;
}
- if (numValue instanceof Number) {
- return ((Number) numValue).longValue();
- } else {
- return Long.parseLong(String.valueOf(numValue));
- }
+ return coerceNumberId(numValue);
case CUSTOMIZE_UUID:
List<Object> uuidValues = getFieldValues(row, idFields);
if (uuidValues.size() != 1) {
@@ -184,16 +300,16 @@ public class VertexMapper implements GraphDataMapper {
private List<Object> getFieldValues(SeaTunnelRow row, List<String> fields) {
List<Object> values = new ArrayList<>(fields.size());
- Map<String, String> fieldMapping = mappingConfig.getFieldMapping();
+ Map<String, String> fm = mappingConfig.getFieldMapping();
for (String fieldName : fields) {
-
Integer index = fieldsIndex.get(fieldName);
if (index == null) {
throw new HugeGraphConnectorException(
HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
String.format(
- "Field '%s' specified in id_fields not found in row schema. Available fields: %s",
- fieldName, fieldsIndex.keySet()));
+ "Mapping[VERTEX/%s]: Field '%s' specified in idFields not found in row schema. "
+ + "Available fields: %s",
+ mappingConfig.getLabel(), fieldName, fieldsIndex.keySet()));
}
Object rawValue = row.getField(index);
@@ -201,17 +317,21 @@ public class VertexMapper implements GraphDataMapper {
continue;
}
- String propertyName = fieldMapping.getOrDefault(fieldName, fieldName);
- PropertyKey propertyKey = propertyKeyCache.get(propertyName);
-
- Object fieldValue =
- DataTypeUtil.convert(
- rawValue,
- propertyKey,
- mappingConfig.getDateFormat(),
- mappingConfig.getTimeZone());
+ String propName = fm.getOrDefault(fieldName, fieldName);
+ PropertyKey propertyKey = propertyKeyCache.get(propName);
- values.add(getMappedValue(fieldValue));
+ Object converted = rawValue;
+ if (propertyKey != null) {
+ converted =
+ DataTypeUtil.convert(
+ rawValue,
+ propertyKey,
+ mappingConfig.getDateFormat(),
+ mappingConfig.getTimeZone(),
+ mappingConfig.getExtraDateFormats(),
+ mappingConfig.getListFormat());
+ }
+ values.add(getMappedValue(fieldName, converted));
}
return values;
}
@@ -221,23 +341,102 @@ public class VertexMapper implements GraphDataMapper {
return true;
}
List<String> nullValues = mappingConfig.getNullValues();
- if (nullValues == null || nullValues.isEmpty()) {
- return false;
- }
- return nullValues.contains(String.valueOf(value));
+ return !nullValues.isEmpty() && nullValues.contains(String.valueOf(value));
}
- private Object getMappedValue(Object originalValue) {
- Map<Object, Object> valueMapping = mappingConfig.getValueMapping();
- if (valueMapping.isEmpty()) {
+ private Object getMappedValue(String sourceField, Object originalValue) {
+ Map<String, Map<Object, Object>> vm = mappingConfig.getValueMapping();
+ if (vm.isEmpty()) {
+ return originalValue;
+ }
+ Map<Object, Object> perField = vm.get(sourceField);
+ if (perField == null || perField.isEmpty()) {
return originalValue;
}
- return valueMapping.getOrDefault(originalValue, originalValue);
+ return perField.getOrDefault(originalValue, originalValue);
}
private String spliceVertexId(List<Object> primaryValues) {
- String joinedValues =
- primaryValues.stream().map(Object::toString).collect(Collectors.joining("!"));
- return String.format("%s:%s", labelId, joinedValues);
+ // HugeGraph primary-key vertex id = {vertexLabelId}:{concatValues(pk)}; concatValues joins
+ // with '!' and backtick-escapes any '!' in a value, matching how the server assembles the
+ // id. EdgeMapper uses the same helper for the same concept — a raw join here would produce
+ // an ambiguous, server-mismatched id when a pk value contains '!', so DELETE / key-changing
+ // UPDATE would target the wrong (or a non-existent) vertex.
+ return checkVertexIdLength(
+ String.format("%s:%s", labelId, SplicingIdGenerator.concatValues(primaryValues)));
+ }
+
+ /**
+ * Assembles a CUSTOMIZE_STRING id from its id-field values. A single field is used verbatim —
+ * it is unambiguous, and escaping it would change ids already written for the common
+ * single-field case. Multiple fields are joined with ':' after backslash-escaping any ':' (and
+ * the '\' escape char itself) in each value, so distinct field tuples cannot collapse to the
+ * same id — e.g. ("x:y","z") and ("x","y:z") no longer both yield "x:y:z" and overwrite each
+ * other. Shared by {@link VertexMapper} and {@code EdgeMapper} so both build ids identically.
+ */
+ static String spliceCustomizeStringId(List<Object> values) {
+ if (values.size() == 1) {
+ return checkVertexIdLength(String.valueOf(values.get(0)));
+ }
+ return checkVertexIdLength(
+ values.stream()
+ .map(String::valueOf)
+ .map(VertexMapper::escapeIdSegment)
+ .collect(Collectors.joining(":")));
+ }
+
+ private static String escapeIdSegment(String value) {
+ return value.replace("\\", "\\\\").replace(":", "\\:");
+ }
+
+ /**
+ * Coerces a CUSTOMIZE_NUMBER id value to a long, consistently for both Number and String
+ * inputs. A fractional value ({@code 1.9} whether it arrives as a Double or the string "1.9")
+ * is rejected rather than silently truncated to {@code 1}; an integral decimal like {@code
+ * 1.0}/"1.0" is accepted. Shared by {@link VertexMapper} and {@code EdgeMapper} so both behave
+ * identically.
+ */
+ static long coerceNumberId(Object value) {
+ java.math.BigDecimal decimal;
+ try {
+ decimal = new java.math.BigDecimal(String.valueOf(value).trim());
+ } catch (NumberFormatException e) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format("CUSTOMIZE_NUMBER id value '%s' is not a number.", value),
+ e);
+ }
+ try {
+ return decimal.longValueExact();
+ } catch (ArithmeticException e) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "CUSTOMIZE_NUMBER id value '%s' is not an integer: a fractional or "
+ + "out-of-range value cannot be used as a numeric id (it would "
+ + "otherwise be silently truncated).",
+ value),
+ e);
+ }
+ }
+
+ /** HugeGraph server per-vertex id cap (see loader Constants.VERTEX_ID_LIMIT). */
+ static final int VERTEX_ID_LIMIT = 128;
+
+ /**
+ * Rejects a string vertex id whose UTF-8 length exceeds the server limit, so the user gets a
+ * clear client-side error instead of an opaque server rejection. Number/UUID ids are always
+ * within the limit and are not checked.
+ */
+ static String checkVertexIdLength(String id) {
+ int length = id.getBytes(StandardCharsets.UTF_8).length;
+ if (length > VERTEX_ID_LIMIT) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "The vertex id length (%d bytes) exceeds the limit of %d: '%s'",
+ length, VERTEX_ID_LIMIT, id));
+ }
+ return id;
}
}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/sink/HugeGraphSaveModeHandler.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/sink/HugeGraphSaveModeHandler.java
new file mode 100644
index 0000000000..e53d847c1a
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/sink/HugeGraphSaveModeHandler.java
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.hugegraph.sink;
+
+import org.apache.seatunnel.api.sink.DataSaveMode;
+import org.apache.seatunnel.api.sink.SaveModeHandler;
+import org.apache.seatunnel.api.sink.SchemaSaveMode;
+import org.apache.seatunnel.api.table.catalog.Catalog;
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.catalog.TablePath;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.client.HugeGraphClient;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.HugeGraphDataSaveMode;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.HugeGraphSchemaSaveMode;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.HugeGraphSinkConfig;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.MappingConfig;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorErrorCode;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorException;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.utils.SchemaManager;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.utils.SchemaValidator;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Handles HugeGraph schema and data save modes on the coordinator, once per job, via the engine's
+ * {@link SaveModeHandler} contract. Doing this here (instead of in the {@code HugeGraphSink}
+ * constructor) is what makes it correct on restart and for multi-table sinks:
+ *
+ * <ul>
+ * <li><b>Restart</b>: on checkpoint restore the engine calls only {@link
+ * #handleSchemaSaveModeWithRestore()}, so data is never dropped a second time — previously
+ * the constructor re-ran the drop on every restart and lost data written before the restart.
+ * <li><b>Multi-table</b>: each table's sink gets its own handler and drops only the labels that
+ * table targets ({@link #handleDataSaveMode()}), so dropping table A no longer wipes table B
+ * — the old whole-graph {@code clearGraph} cleared everything (and destroyed a sibling
+ * table's freshly-created schema).
+ * </ul>
+ *
+ * Schema work runs before the data drop (the default {@link #handleSaveMode()} order) so the labels
+ * exist when their data is cleared.
+ */
+public class HugeGraphSaveModeHandler implements SaveModeHandler {
+
+ private final HugeGraphSinkConfig config;
+ private final SeaTunnelRowType rowType;
+ private final TablePath tablePath;
+
+ private HugeGraphClient client;
+
+ public HugeGraphSaveModeHandler(
+ HugeGraphSinkConfig config, SeaTunnelRowType rowType, TablePath tablePath) {
+ this.config = config;
+ this.rowType = rowType;
+ this.tablePath = tablePath;
+ }
+
+ @Override
+ public void open() {
+ this.client = createClient();
+ }
+
+ /** Test seam: overridden in unit tests to inject a mock client instead of a live connection. */
+ HugeGraphClient createClient() {
+ return new HugeGraphClient(config.getConnectionConfig());
+ }
+
+ /**
+ * Config-level checks (labels, idFields, MULTIPLE→sortKeys, source-field presence) run before
+ * any server write so a malformed mapping cannot leave a partial schema behind — the HugeGraph
+ * server is non-transactional for DDL and its primary keys / sort keys / frequency are
+ * effectively immutable. Under CREATE_SCHEMA_WHEN_NOT_EXIST, missing PropertyKey / VertexLabel
+ * / EdgeLabel are then auto-created and finally re-validated against the server. Under
+ * ERROR_WHEN_SCHEMA_NOT_EXIST, only validation runs.
+ */
+ @Override
+ public void handleSchemaSaveMode() {
+ SchemaValidator validator = new SchemaValidator(client, rowType);
+ validator.validateConfigOnly(config.getMappings());
+
+ if (config.getSchemaSaveMode() == HugeGraphSchemaSaveMode.CREATE_SCHEMA_WHEN_NOT_EXIST) {
+ // Fail fast on any already-existing label whose immutable attributes (PK, frequency,
+ // sort keys, endpoints) conflict with the config, BEFORE creating anything — a conflict
+ // discovered only by the post-create validate() below would leave the PropertyKeys /
+ // labels created for the other mappings behind as schema pollution.
+ validator.validateExistingLabels(config.getMappings());
+
+ SchemaManager schemaManager =
+ new SchemaManager(client, config.getSchemaSaveMode(), rowType);
+ schemaManager.ensureSchema(config.getMappings());
+ }
+
+ validator.validate(config.getMappings());
+ }
+
+ /**
+ * For DROP_DATA, clear only the data of the labels this job's mappings target — edges first (so
+ * edge-only mappings are handled even when their endpoints are out of scope), then vertices
+ * (removing a vertex also removes its remaining incident edges). Schema is preserved.
+ * APPEND_DATA is a no-op.
+ *
+ * <p>Before deleting vertices, a pre-flight check discovers every edge label that references a
+ * target vertex label. If any of those edge labels are NOT in this job's mappings, the job
+ * fails fast — deleting the vertices would cascade-delete those edges silently. Set {@code
+ * allow_cascade_delete_unmapped_edges=true} to opt into the destructive cascade.
+ */
+ @Override
+ public void handleDataSaveMode() {
+ if (config.getDataSaveMode() != HugeGraphDataSaveMode.DROP_DATA) {
+ return;
+ }
+ List<MappingConfig> mappings = config.getMappings();
+
+ // Collect the set of edge labels this job explicitly targets.
+ Set<String> mappedEdgeLabels = new HashSet<>();
+ Set<String> mappedVertexLabels = new HashSet<>();
+ for (MappingConfig mapping : mappings) {
+ if (mapping.getType() == MappingConfig.LabelType.EDGE) {
+ mappedEdgeLabels.add(mapping.getLabel());
+ } else {
+ mappedVertexLabels.add(mapping.getLabel());
+ }
+ }
+
+ // Pre-flight: for each vertex label being dropped, discover edge labels that would be
+ // cascade-deleted. If any are not in this job's mappings, fail fast — unless the user
+ // has explicitly opted into the destructive cascade.
+ if (!config.isAllowCascadeDeleteUnmappedEdges()) {
+ for (String vertexLabel : mappedVertexLabels) {
+ List<String> connected = client.getConnectedEdgeLabels(vertexLabel);
+ for (String edgeLabel : connected) {
+ if (!mappedEdgeLabels.contains(edgeLabel)) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "DROP_DATA would cascade-delete edge label '%s' (connected to "
+ + "vertex label '%s'), which is not in this job's "
+ + "mappings. Add '%s' to your mappings to delete it "
+ + "explicitly, or set "
+ + "allow_cascade_delete_unmapped_edges=true to accept "
+ + "the destructive cascade.",
+ edgeLabel, vertexLabel, edgeLabel));
+ }
+ }
+ }
+ }
+
+ for (MappingConfig mapping : mappings) {
+ if (mapping.getType() == MappingConfig.LabelType.EDGE) {
+ client.deleteEdgesByLabel(mapping.getLabel());
+ }
+ }
+ for (MappingConfig mapping : mappings) {
+ if (mapping.getType() == MappingConfig.LabelType.VERTEX) {
+ client.deleteVerticesByLabel(mapping.getLabel());
+ }
+ }
+ }
+
+ /**
+ * Restore path: (re)ensure and validate schema only. Deliberately never drops data — the data
+ * already written before the checkpoint must survive the restart.
+ */
+ @Override
+ public void handleSchemaSaveModeWithRestore() {
+ handleSchemaSaveMode();
+ }
+
+ @Override
+ public SchemaSaveMode getSchemaSaveMode() {
+ return config.getSchemaSaveMode() == HugeGraphSchemaSaveMode.CREATE_SCHEMA_WHEN_NOT_EXIST
+ ? SchemaSaveMode.CREATE_SCHEMA_WHEN_NOT_EXIST
+ : SchemaSaveMode.ERROR_WHEN_SCHEMA_NOT_EXIST;
+ }
+
+ @Override
+ public DataSaveMode getDataSaveMode() {
+ return config.getDataSaveMode() == HugeGraphDataSaveMode.DROP_DATA
+ ? DataSaveMode.DROP_DATA
+ : DataSaveMode.APPEND_DATA;
+ }
+
+ @Override
+ public TablePath getHandleTablePath() {
+ return tablePath;
+ }
+
+ @Override
+ public Catalog getHandleCatalog() {
+ // HugeGraph has no SeaTunnel Catalog implementation; schema/data handling goes through the
+ // HugeGraph client directly. The engine's SaveModeExecuteWrapper only reads name() from
+ // this
+ // for logging, so a lightweight stub is sufficient.
+ return new HugeGraphNamedCatalog();
+ }
+
+ @Override
+ public void close() {
+ if (client != null) {
+ client.close();
+ client = null;
+ }
+ }
+
+ /**
+ * Minimal {@link Catalog} that exists only to satisfy {@code SaveModeExecuteWrapper}, which
+ * logs {@code getHandleCatalog().name()} before running the handler. HugeGraph does all
+ * schema/data work through its own client, so every catalog operation other than {@link
+ * #name()} is unsupported and never invoked on the save-mode path.
+ */
+ private static final class HugeGraphNamedCatalog implements Catalog {
+
+ @Override
+ public String name() {
+ return "HugeGraph";
+ }
+
+ @Override
+ public void open() {}
+
+ @Override
+ public void close() {}
+
+ @Override
+ public String getDefaultDatabase() {
+ throw unsupported();
+ }
+
+ @Override
+ public boolean databaseExists(String databaseName) {
+ throw unsupported();
+ }
+
+ @Override
+ public List<String> listDatabases() {
+ throw unsupported();
+ }
+
+ @Override
+ public List<String> listTables(String databaseName) {
+ throw unsupported();
+ }
+
+ @Override
+ public boolean tableExists(TablePath tablePath) {
+ throw unsupported();
+ }
+
+ @Override
+ public CatalogTable getTable(TablePath tablePath) {
+ throw unsupported();
+ }
+
+ @Override
+ public void createTable(TablePath tablePath, CatalogTable table, boolean ignoreIfExists) {
+ throw unsupported();
+ }
+
+ @Override
+ public void dropTable(TablePath tablePath, boolean ignoreIfNotExists) {
+ throw unsupported();
+ }
+
+ @Override
+ public void createDatabase(TablePath tablePath, boolean ignoreIfExists) {
+ throw unsupported();
+ }
+
+ @Override
+ public void dropDatabase(TablePath tablePath, boolean ignoreIfNotExists) {
+ throw unsupported();
+ }
+
+ private static UnsupportedOperationException unsupported() {
+ return new UnsupportedOperationException(
+ "HugeGraph does not provide a SeaTunnel Catalog; "
+ + "schema and data save modes are handled via the HugeGraph client.");
+ }
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/sink/HugeGraphSink.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/sink/HugeGraphSink.java
index 91f594a04b..da6227ec20 100644
--- a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/sink/HugeGraphSink.java
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/sink/HugeGraphSink.java
@@ -17,32 +17,48 @@
package org.apache.seatunnel.connectors.seatunnel.hugegraph.sink;
+import org.apache.seatunnel.api.sink.SaveModeHandler;
import org.apache.seatunnel.api.sink.SinkWriter;
+import org.apache.seatunnel.api.sink.SupportMultiTableSink;
+import org.apache.seatunnel.api.sink.SupportSaveMode;
import org.apache.seatunnel.api.table.catalog.CatalogTable;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
import org.apache.seatunnel.connectors.seatunnel.common.sink.AbstractSimpleSink;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.HugeGraphOptions;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.HugeGraphSinkConfig;
-import org.apache.seatunnel.connectors.seatunnel.hugegraph.utils.SchemaValidator;
import java.io.IOException;
import java.util.Optional;
-public class HugeGraphSink extends AbstractSimpleSink<SeaTunnelRow, Void> {
+public class HugeGraphSink extends AbstractSimpleSink<SeaTunnelRow, Void>
+ implements SupportMultiTableSink, SupportSaveMode {
private final HugeGraphSinkConfig config;
private final CatalogTable catalogTable;
private final SeaTunnelRowType rowType;
+ private final String tablePath;
public HugeGraphSink(HugeGraphSinkConfig config, CatalogTable catalogTable) {
this.config = config;
this.catalogTable = catalogTable;
this.rowType = catalogTable.getSeaTunnelRowType();
+ this.tablePath = catalogTable.getTablePath().toString();
- // TODO: Discuss where to implement this in the future, maybe the catalog
- SchemaValidator validator = new SchemaValidator(config, rowType);
- validator.validateSchema();
+ this.config.applyLegacyFieldSelection(rowType);
+ }
+
+ /**
+ * Schema management and the DROP_DATA data drop run once on the coordinator via the engine's
+ * SaveMode contract — see {@link HugeGraphSaveModeHandler}. Running it here (rather than in the
+ * constructor as before) is what makes it correct on checkpoint restart and for multi-table
+ * sinks: restart re-runs only the schema step (never dropping data), and each table drops only
+ * its own labels instead of wiping the whole graph.
+ */
+ @Override
+ public Optional<SaveModeHandler> getSaveModeHandler() {
+ return Optional.of(
+ new HugeGraphSaveModeHandler(config, rowType, catalogTable.getTablePath()));
}
@Override
@@ -52,7 +68,7 @@ public class HugeGraphSink extends AbstractSimpleSink<SeaTunnelRow, Void> {
@Override
public HugeGraphSinkWriter createWriter(SinkWriter.Context context) throws IOException {
- return new HugeGraphSinkWriter(config, rowType);
+ return new HugeGraphSinkWriter(config, rowType, tablePath, context.getIndexOfSubtask());
}
@Override
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/sink/HugeGraphSinkFactory.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/sink/HugeGraphSinkFactory.java
index 56e75ab214..fa4630a558 100644
--- a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/sink/HugeGraphSinkFactory.java
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/sink/HugeGraphSinkFactory.java
@@ -18,6 +18,7 @@
package org.apache.seatunnel.connectors.seatunnel.hugegraph.sink;
import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.api.options.SinkConnectorCommonOptions;
import org.apache.seatunnel.api.table.connector.TableSink;
import org.apache.seatunnel.api.table.factory.Factory;
import org.apache.seatunnel.api.table.factory.TableSinkFactory;
@@ -48,17 +49,40 @@ public class HugeGraphSinkFactory implements TableSinkFactory {
// connection config
.required(HugeGraphOptions.HOST, HugeGraphOptions.PORT, HugeGraphOptions.GRAPH_NAME)
.optional(
- HugeGraphOptions.GRAPH_SPACE,
+ HugeGraphOptions.PROTOCOL,
HugeGraphOptions.USERNAME,
- HugeGraphOptions.PASSWORD)
- // mapping config
- .exclusive(
- HugeGraphSinkOptions.SELECTED_FIELDS, HugeGraphSinkOptions.IGNORED_FIELDS)
- .required(HugeGraphSinkOptions.SCHEMA_CONFIG)
+ HugeGraphOptions.PASSWORD,
+ // Optional connection setting passed through to select the HugeGraph graph
+ // space (defaults to "DEFAULT").
+ HugeGraphOptions.GRAPH_SPACE)
+ // mapping config: mappings (new) or schema_config (legacy)
+ .optional(HugeGraphSinkOptions.MAPPINGS, HugeGraphSinkOptions.SCHEMA_CONFIG)
+ // schema and data save mode
+ .optional(
+ HugeGraphSinkOptions.SCHEMA_SAVE_MODE,
+ HugeGraphSinkOptions.DATA_SAVE_MODE,
+ HugeGraphSinkOptions.DELETE_VERTEX_WITH_EDGES,
+ HugeGraphSinkOptions.ALLOW_CASCADE_DELETE_UNMAPPED_EDGES)
// batch config
- .optional(HugeGraphOptions.BATCH_SIZE, HugeGraphOptions.BATCH_INTERVAL_MS)
- // error operation
- .optional(HugeGraphOptions.MAX_RETRIES, HugeGraphOptions.RETRY_BACKOFF_MS)
+ .optional(
+ HugeGraphOptions.BATCH_SIZE,
+ HugeGraphOptions.BATCH_INTERVAL_MS,
+ HugeGraphOptions.CHECK_VERTEX)
+ // required by the multi-table sink SPI (HugeGraphSink implements
+ // SupportMultiTableSink): lets the framework size per-table write replicas
+ .optional(SinkConnectorCommonOptions.MULTI_TABLE_SINK_REPLICA)
+ // error handling
+ .optional(
+ HugeGraphOptions.BATCH_FAILURE_FALLBACK,
+ HugeGraphOptions.MAX_INSERT_ERRORS,
+ HugeGraphOptions.FAILURE_DATA_PATH)
+ // retry config
+ .optional(
+ HugeGraphOptions.MAX_RETRIES,
+ HugeGraphOptions.RETRY_BACKOFF_MS,
+ HugeGraphOptions.RETRY_BACKOFF_MAX_MS)
+ // deprecated field selection
+ .optional(HugeGraphSinkOptions.SELECTED_FIELDS, HugeGraphSinkOptions.IGNORED_FIELDS)
.build();
}
}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/sink/HugeGraphSinkWriter.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/sink/HugeGraphSinkWriter.java
index 2a71ceba1e..cbad8e1ec5 100644
--- a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/sink/HugeGraphSinkWriter.java
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/sink/HugeGraphSinkWriter.java
@@ -22,10 +22,11 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
import org.apache.seatunnel.connectors.seatunnel.common.sink.AbstractSinkWriter;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.buffer.BatchBuffer;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.buffer.GraphElementEnvelope;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.client.HugeGraphClient;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.HugeGraphSinkConfig;
-import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.SchemaConfig;
-import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.SchemaConfig.LabelType;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.MappingConfig;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.MappingConfig.LabelType;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorErrorCode;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorException;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.mapper.EdgeMapper;
@@ -38,7 +39,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashSet;
+import java.util.IdentityHashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -53,55 +57,264 @@ public class HugeGraphSinkWriter extends AbstractSinkWriter<SeaTunnelRow, Void>
private static final Logger LOG = LoggerFactory.getLogger(HugeGraphSinkWriter.class);
private final HugeGraphSinkConfig sinkConfig;
- private final GraphDataMapper mapper;
+ private final List<MappingEntry> mappingEntries;
private final HugeGraphClient client;
private final BatchBuffer buffer;
+ private final String tablePath;
+
+ // Holds the UPDATE_BEFORE row until its paired UPDATE_AFTER arrives (changelog streams
+ // deliver them consecutively). This field is NOT checkpointable — the SeaTunnel SinkWriter
+ // interface has no snapshotState(). If a checkpoint fires between UPDATE_BEFORE and
+ // UPDATE_AFTER, prepareCommit() fails fast rather than risking a partially-applied mutation.
+ private SeaTunnelRow pendingUpdateBefore;
public HugeGraphSinkWriter(HugeGraphSinkConfig sinkConfig, SeaTunnelRowType rowType) {
+ this(sinkConfig, rowType, "", 0);
+ }
+
+ public HugeGraphSinkWriter(
+ HugeGraphSinkConfig sinkConfig, SeaTunnelRowType rowType, int subtaskIndex) {
+ this(sinkConfig, rowType, "", subtaskIndex);
+ }
+
+ public HugeGraphSinkWriter(
+ HugeGraphSinkConfig sinkConfig,
+ SeaTunnelRowType rowType,
+ String tablePath,
+ int subtaskIndex) {
+ this(
+ sinkConfig,
+ rowType,
+ tablePath,
+ new HugeGraphClient(sinkConfig.getConnectionConfig()),
+ subtaskIndex);
+ }
+
+ HugeGraphSinkWriter(
+ HugeGraphSinkConfig sinkConfig, SeaTunnelRowType rowType, HugeGraphClient client) {
+ this(sinkConfig, rowType, "", client, 0);
+ }
+
+ HugeGraphSinkWriter(
+ HugeGraphSinkConfig sinkConfig,
+ SeaTunnelRowType rowType,
+ HugeGraphClient client,
+ int subtaskIndex) {
+ this(sinkConfig, rowType, "", client, subtaskIndex);
+ }
+
+ HugeGraphSinkWriter(
+ HugeGraphSinkConfig sinkConfig,
+ SeaTunnelRowType rowType,
+ String tablePath,
+ HugeGraphClient client,
+ int subtaskIndex) {
this.sinkConfig = sinkConfig;
- this.client = new HugeGraphClient(sinkConfig);
- this.mapper = getMapper(rowType);
+ this.tablePath = tablePath;
+ this.sinkConfig.applyLegacyFieldSelection(rowType);
+ this.client = client;
+ try {
+ // buildMappingEntries issues live schema lookups; if any fails the framework will not
+ // call close() on this half-constructed writer, so release the client here.
+ this.mappingEntries = buildMappingEntries(rowType);
+ } catch (RuntimeException e) {
+ try {
+ this.client.close();
+ } catch (RuntimeException closeFailure) {
+ e.addSuppressed(closeFailure);
+ }
+ throw e;
+ }
this.buffer =
new BatchBuffer(
- this.client, sinkConfig.getBatchSize(), sinkConfig.getBatchIntervalMs());
+ this.client,
+ sinkConfig.getBatchSize(),
+ sinkConfig.getBatchIntervalMs(),
+ sinkConfig.isBatchFailureFallback(),
+ sinkConfig.isCheckVertex(),
+ sinkConfig.getMaxInsertErrors(),
+ sinkConfig.getFailureDataPath(),
+ subtaskIndex);
}
- private GraphDataMapper getMapper(SeaTunnelRowType rowType) {
- SchemaConfig schemaConfig = sinkConfig.getSchemaConfig();
- List<String> selectedFields = sinkConfig.getSelectedFields();
- List<String> ignoredFields = sinkConfig.getIgnoredFields();
+ private List<MappingEntry> buildMappingEntries(SeaTunnelRowType rowType) {
Map<String, Integer> originalFieldsIndex =
IntStream.range(0, rowType.getTotalFields())
.boxed()
- .collect(Collectors.toMap(rowType::getFieldName, i -> i));
+ .collect(
+ Collectors.toMap(
+ rowType::getFieldName,
+ i -> i,
+ (a, b) -> a,
+ LinkedHashMap::new));
+ Map<String, Integer> availableFieldsIndex = resolveLegacyFieldsIndex(originalFieldsIndex);
+
+ List<MappingEntry> entries = new ArrayList<>();
+ for (MappingConfig mapping : sinkConfig.getMappings()) {
+ // Multi-table binding: when a mapping declares source_table, only activate it in the
+ // writer whose tablePath matches. A mapping without source_table activates in every
+ // writer — the single-table backward-compatible default.
+ if (!mapping.appliesTo(tablePath)) {
+ LOG.info(
+ "Mapping[{}/{}] source_table '{}' does not match writer table '{}'; "
+ + "skipping in this writer.",
+ mapping.getType(),
+ mapping.getLabel(),
+ mapping.getSourceTable(),
+ tablePath);
+ continue;
+ }
+ Map<String, Integer> fieldsIndex = resolveFieldsIndex(mapping, availableFieldsIndex);
+ GraphDataMapper mapper;
+ if (mapping.getType() == LabelType.VERTEX) {
+ if (mapping.getIdStrategy()
+ == org.apache.hugegraph.structure.constant.IdStrategy.AUTOMATIC) {
+ // AUTOMATIC ids are server-assigned and not derivable from the row, so there is
+ // no key to deduplicate on: under at-least-once, a replayed row inserts a NEW
+ // vertex each time (duplicates). Warn so the trade-off is visible; use
+ // PRIMARY_KEY / CUSTOMIZE_* for idempotent upserts.
+ LOG.warn(
+ "Mapping[VERTEX/{}] uses AUTOMATIC ids: under at-least-once delivery a "
+ + "replayed row creates a duplicate vertex, and DELETE is not "
+ + "supported. Use PRIMARY_KEY or CUSTOMIZE_* ids for idempotent "
+ + "writes.",
+ mapping.getLabel());
+ }
+ mapper = new VertexMapper(mapping, fieldsIndex, client);
+ } else {
+ mapper = new EdgeMapper(mapping, fieldsIndex, client);
+ }
+ entries.add(new MappingEntry(mapping, mapper));
+ }
- Map<String, Integer> finalFieldsIndex = new LinkedHashMap<>();
+ if (entries.isEmpty()) {
+ // Multi-table mode: at least one mapping has source_table, but none matched this
+ // writer's tablePath. This is a configuration error — the user intended multi-table
+ // but the table path strings don't align. Fail fast with a diagnostic that shows
+ // both sides so the user can reconcile them.
+ boolean multiTable =
+ sinkConfig.getMappings().stream()
+ .anyMatch(
+ m ->
+ m.getSourceTable() != null
+ && !m.getSourceTable().isEmpty());
+ if (multiTable) {
+ List<String> configuredTables =
+ sinkConfig.getMappings().stream()
+ .map(MappingConfig::getSourceTable)
+ .filter(t -> t != null && !t.isEmpty())
+ .distinct()
+ .collect(Collectors.toList());
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "No mapping matched writer table '%s'. The configured "
+ + "source_table value(s) are %s. Verify that each "
+ + "source_table matches the CatalogTable.getTablePath() "
+ + "produced by the upstream Source (in read-all mode this "
+ + "is the HugeGraph label name).",
+ tablePath, configuredTables));
+ }
+ // Single-table mode (no source_table set on any mapping): no entries means the
+ // row type carries no fields the mappings can use. This is unusual but not
+ // necessarily a config error — a downstream project may legitimately sink only a
+ // subset of fields. Log and continue as a no-op.
+ LOG.warn(
+ "No mappings matched table '{}' in this writer. The writer will be a no-op: "
+ + "rows are acknowledged without writing to HugeGraph.",
+ tablePath);
+ }
+ return entries;
+ }
+
+ private Map<String, Integer> resolveLegacyFieldsIndex(
+ Map<String, Integer> originalFieldsIndex) {
+ if (sinkConfig.getSchemaConfig() == null) {
+ return originalFieldsIndex;
+ }
+
+ List<String> selectedFields = sinkConfig.getSelectedFields();
if (selectedFields != null && !selectedFields.isEmpty()) {
+ Map<String, Integer> selected = new LinkedHashMap<>();
for (String field : selectedFields) {
- Integer originalIndex = originalFieldsIndex.get(field);
- if (originalIndex != null) {
- finalFieldsIndex.put(field, originalIndex);
+ Integer index = originalFieldsIndex.get(field);
+ if (index != null) {
+ selected.put(field, index);
}
}
- } else if (ignoredFields != null && !ignoredFields.isEmpty()) {
- Set<String> ignoreSet = new HashSet<>(ignoredFields);
+ return selected;
+ }
+
+ List<String> ignoredFields = sinkConfig.getIgnoredFields();
+ if (ignoredFields != null && !ignoredFields.isEmpty()) {
+ Set<String> ignored = new HashSet<>(ignoredFields);
+ Map<String, Integer> selected = new LinkedHashMap<>();
for (Map.Entry<String, Integer> entry : originalFieldsIndex.entrySet()) {
- String fieldName = entry.getKey();
- Integer originalIndex = entry.getValue();
+ if (!ignored.contains(entry.getKey())) {
+ selected.put(entry.getKey(), entry.getValue());
+ }
+ }
+ return selected;
+ }
+ return originalFieldsIndex;
+ }
- if (!ignoreSet.contains(fieldName)) {
- finalFieldsIndex.put(fieldName, originalIndex);
+ private Map<String, Integer> resolveFieldsIndex(
+ MappingConfig mapping, Map<String, Integer> originalFieldsIndex) {
+ // If no explicit properties, use all fields from the row
+ if (mapping.getProperties().isEmpty()) {
+ return new LinkedHashMap<>(originalFieldsIndex);
+ }
+
+ // Build index from explicit properties + id fields
+ Map<String, Integer> result = new LinkedHashMap<>();
+
+ for (String field : mapping.getProperties()) {
+ Integer idx = originalFieldsIndex.get(field);
+ if (idx != null) {
+ result.put(field, idx);
+ }
+ }
+
+ // New mappings always include fields required to build IDs. Legacy selected_fields keeps
+ // its original strict filtering behavior for backward compatibility.
+ if (sinkConfig.getSchemaConfig() == null && mapping.getIdFields() != null) {
+ for (String field : mapping.getIdFields()) {
+ Integer idx = originalFieldsIndex.get(field);
+ if (idx != null) {
+ result.put(field, idx);
}
}
- } else {
- finalFieldsIndex = originalFieldsIndex;
}
- if (schemaConfig.getType() == LabelType.VERTEX) {
- return new VertexMapper(schemaConfig, finalFieldsIndex, client);
- } else {
- return new EdgeMapper(schemaConfig, finalFieldsIndex, client);
+ // For edges, include source/target idFields and sortKeys
+ if (sinkConfig.getSchemaConfig() == null && mapping.getType() == LabelType.EDGE) {
+ includeEdgeIdFields(mapping.getSourceConfig(), originalFieldsIndex, result);
+ includeEdgeIdFields(mapping.getTargetConfig(), originalFieldsIndex, result);
+
+ for (String field : mapping.getSortKeys()) {
+ Integer idx = originalFieldsIndex.get(field);
+ if (idx != null) {
+ result.put(field, idx);
+ }
+ }
+ }
+
+ return result;
+ }
+
+ private void includeEdgeIdFields(
+ MappingConfig.SourceTargetConfig stConfig,
+ Map<String, Integer> originalFieldsIndex,
+ Map<String, Integer> result) {
+ if (stConfig != null && stConfig.getIdFields() != null) {
+ for (String field : stConfig.getIdFields()) {
+ Integer idx = originalFieldsIndex.get(field);
+ if (idx != null) {
+ result.put(field, idx);
+ }
+ }
}
}
@@ -109,15 +322,20 @@ public class HugeGraphSinkWriter extends AbstractSinkWriter<SeaTunnelRow, Void>
public void write(SeaTunnelRow row) throws IOException {
switch (row.getRowKind()) {
case INSERT:
- case UPDATE_AFTER:
handleUpsert(row);
break;
+ case UPDATE_AFTER:
+ handleUpdate(pendingUpdateBefore, row);
+ pendingUpdateBefore = null;
+ break;
case DELETE:
handleDelete(row);
break;
case UPDATE_BEFORE:
- // The huge-client natively supports upsert operations for property updates, so
- // there is no need to handle this data manually.
+ // Correlated with the immediately following UPDATE_AFTER (changelog contract) and
+ // handled together in handleUpdate, so a key-changing update deletes the
+ // pre-update element instead of leaving it orphaned.
+ pendingUpdateBefore = row;
break;
default:
LOG.warn("Unsupported row kind: {}", row.getRowKind());
@@ -126,52 +344,446 @@ public class HugeGraphSinkWriter extends AbstractSinkWriter<SeaTunnelRow, Void>
}
private void handleUpsert(SeaTunnelRow row) throws IOException {
- try {
- GraphElement element = mapper.map(row);
- if (element == null) {
- LOG.warn("Cannot create graph element: required ID fields missing for row {}", row);
- return;
+ List<GraphElementEnvelope> vertexEnvelopes = new ArrayList<>();
+ List<GraphElementEnvelope> edgeEnvelopes = new ArrayList<>();
+
+ for (MappingEntry entry : mappingEntries) {
+ // mapToEnvelopes returns 1 element normally, or N when unfold expands a list cell.
+ for (GraphElementEnvelope envelope : mapToEnvelopes(entry, row)) {
+ if (entry.config.getType() == LabelType.VERTEX) {
+ vertexEnvelopes.add(envelope);
+ } else {
+ edgeEnvelopes.add(envelope);
+ }
}
- buffer.add(element);
- } catch (Exception e) {
- if (e instanceof IOException) {
- throw (IOException) e;
+ }
+
+ for (GraphElementEnvelope envelope : vertexEnvelopes) {
+ buffer.add(envelope);
+ }
+ for (GraphElementEnvelope envelope : edgeEnvelopes) {
+ buffer.add(envelope);
+ }
+ }
+
+ /**
+ * unfold (one row → many elements) is only defined for the append/INSERT path. UPDATE/DELETE
+ * would require diffing N old ids against N new ids per mapping, which is out of scope and
+ * dangerous to get wrong, so reject a changelog row when any mapping enables unfold.
+ */
+ private void rejectUnfoldForChangelog(String rowKind) {
+ for (MappingEntry entry : mappingEntries) {
+ if (entry.mapper.isUnfoldEnabled()) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "Mapping[%s/%s]: unfold is only supported for INSERT/append-only "
+ + "jobs, but received a %s row. Disable unfold or run this "
+ + "as an append-only job.",
+ entry.config.getType(), entry.config.getLabel(), rowKind));
}
- throw new IOException(e);
}
}
- private void handleDelete(SeaTunnelRow row) {
+ /**
+ * AUTOMATIC-id vertices have no client-derivable id (the server assigns it), so a DELETE cannot
+ * identify the target. Reject with a clear message instead of the misleading "required ID field
+ * is null" — there is no id field to be null. Package-private + static so it can be unit-tested
+ * without constructing a real writer/client.
+ */
+ static void rejectAutomaticVertexDelete(List<MappingEntry> mappingEntries) {
+ for (MappingEntry entry : mappingEntries) {
+ if (entry.config.getType() == LabelType.VERTEX
+ && entry.config.getIdStrategy()
+ == org.apache.hugegraph.structure.constant.IdStrategy.AUTOMATIC) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "Mapping[VERTEX/%s]: DELETE is not supported with AUTOMATIC IDs "
+ + "because the target vertex cannot be identified from row "
+ + "data. Use a PRIMARY_KEY or CUSTOMIZE_* id strategy if this "
+ + "stream emits deletes.",
+ entry.config.getLabel()));
+ }
+ }
+ }
+
+ /**
+ * Applies a changelog update transactionally with respect to the after-image validity.
+ *
+ * <p>The previous implementation issued the pre-update delete first and then upserted the
+ * after-image; if the after-image mapping failed (null ID, unsupported type conversion, or an
+ * AUTOMATIC-vertex mapping on update) the old vertex/edge had already been deleted, so the row
+ * was lost. This implementation builds a full replacement plan — envelopes for every mapping's
+ * after-image plus the set of superseded IDs — BEFORE touching the server. Any failure during
+ * plan-building raises without any remote side effect, so the pre-update elements stay intact
+ * and the source can replay the row after the config is fixed.
+ *
+ * <p>A superseded (old) element is deleted only when its mapping also produced a replacement
+ * after-image. If the after-image is absent — the after row cannot be mapped for that mapping,
+ * e.g. a null id field made {@code map()} return null — the old element is left untouched
+ * rather than deleted-with-nothing-written; a real removal must arrive as a DELETE event.
+ *
+ * <p>Note: HugeGraph server DDL is non-transactional across a flush+delete pair, so a crash
+ * between the two still leaves partial state; that is an inherent limitation of the REST API
+ * and is handled by at-least-once replay from the upstream source.
+ */
+ private void handleUpdate(SeaTunnelRow before, SeaTunnelRow after) throws IOException {
+ rejectUnfoldForChangelog("UPDATE");
+ UpdatePlan plan = buildUpdatePlan(mappingEntries, before, after);
+ executeUpdatePlan(plan);
+ }
+
+ /**
+ * Package-private + static so the "no side effect on mapping failure" invariant can be pinned
+ * by a unit test without constructing a real writer/client.
+ */
+ static UpdatePlan buildUpdatePlan(
+ List<MappingEntry> mappingEntries, SeaTunnelRow before, SeaTunnelRow after) {
+ List<GraphElementEnvelope> newVertices = new ArrayList<>();
+ List<GraphElementEnvelope> newEdges = new ArrayList<>();
+ List<Superseded> supersededVertices = new ArrayList<>();
+ List<Superseded> supersededEdges = new ArrayList<>();
+
+ Set<MappingEntry> producedAfterImage = Collections.newSetFromMap(new IdentityHashMap<>());
+ for (MappingEntry entry : mappingEntries) {
+ GraphElementEnvelope envelope = mapToEnvelope(entry, after, true);
+ if (envelope == null) {
+ continue;
+ }
+ producedAfterImage.add(entry);
+ if (entry.config.getType() == LabelType.VERTEX) {
+ newVertices.add(envelope);
+ } else {
+ newEdges.add(envelope);
+ }
+ }
+
+ if (before != null) {
+ for (MappingEntry entry : mappingEntries) {
+ // Only delete the pre-update element when this mapping produced a replacement
+ // after-image. If the after-image is absent (e.g. the after row has a null id
+ // field so map() returned null), deleting the old element would drop it with
+ // nothing written back — a silent data loss. Keeping the old element is the safe
+ // choice; a genuine removal should arrive as a DELETE changelog event.
+ if (!producedAfterImage.contains(entry)) {
+ continue;
+ }
+ Object oldId;
+ Object newId;
+ try {
+ oldId = entry.mapper.extractId(before);
+ newId = entry.mapper.extractId(after);
+ } catch (Exception e) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
+ String.format(
+ "Mapping[%s/%s]: Failed to compute graph element ID for update",
+ entry.config.getType(), entry.config.getLabel()),
+ e);
+ }
+ if (oldId == null || oldId.equals(newId)) {
+ continue;
+ }
+ Superseded s = new Superseded(entry, oldId);
+ if (entry.config.getType() == LabelType.VERTEX) {
+ supersededVertices.add(s);
+ } else {
+ supersededEdges.add(s);
+ }
+ }
+ }
+
+ return new UpdatePlan(newVertices, newEdges, supersededVertices, supersededEdges);
+ }
+
+ private void executeUpdatePlan(UpdatePlan plan) throws IOException {
+ // Buffer new envelopes first — mirrors handleUpsert ordering for INSERT and ensures the
+ // new elements are staged before any destructive operation.
+ for (GraphElementEnvelope envelope : plan.newVertices) {
+ buffer.add(envelope);
+ }
+ for (GraphElementEnvelope envelope : plan.newEdges) {
+ buffer.add(envelope);
+ }
+
+ if (plan.supersededVertices.isEmpty() && plan.supersededEdges.isEmpty()) {
+ // ID unchanged — the upsert alone updates the element in place and a vertex's
+ // adjacent edges are preserved.
+ return;
+ }
+
+ // Persist the new elements before issuing deletes. If flush fails, no delete happens, so
+ // pre-update elements stay intact and the source will replay the row.
+ // NOTE: HugeGraph server DDL is non-transactional across a flush+delete pair, so a crash
+ // between the two leaves partial state; this is an inherent limitation of the REST API
+ // and is handled by at-least-once replay from the upstream source. The SinkWriter
+ // interface has no snapshotState(), so the connector cannot checkpoint the in-flight
+ // mutation. prepareCommit() guards against a pending UPDATE_BEFORE crossing a checkpoint
+ // boundary by failing fast.
try {
buffer.flush();
- if (sinkConfig.getSchemaConfig().getType() == LabelType.VERTEX) {
- Object vertexId = mapper.extractId(row);
- if (vertexId == null) {
- LOG.warn("Cannot delete vertex: ID extraction failed for row {}", row);
- return;
- }
- client.deleteVertexWithEdges(vertexId);
- } else {
- String edgeId = (String) mapper.extractId(row);
- if (edgeId == null) {
- LOG.warn("Cannot delete edge: ID extraction failed for row {}", row);
- return;
+ } catch (IOException e) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
+ "Failed to flush buffer before UPDATE cleanup",
+ e);
+ }
+
+ // Delete edges before vertices for topology safety (mirror handleDelete).
+ for (Superseded s : plan.supersededEdges) {
+ try {
+ client.deleteEdge((String) s.oldId);
+ } catch (Exception e) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
+ String.format(
+ "Mapping[%s/%s]: Failed to delete superseded edge on update",
+ s.entry.config.getType(), s.entry.config.getLabel()),
+ e);
+ }
+ }
+ for (Superseded s : plan.supersededVertices) {
+ try {
+ if (sinkConfig.isDeleteVertexWithEdges()) {
+ client.deleteVertexWithEdges(s.oldId);
+ } else {
+ client.deleteVertex(s.oldId);
}
- client.deleteEdge(edgeId);
+ } catch (Exception e) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
+ String.format(
+ "Mapping[%s/%s]: Failed to delete superseded vertex on update",
+ s.entry.config.getType(), s.entry.config.getLabel()),
+ e);
}
+ }
+ }
+
+ /**
+ * Maps a row into an envelope for one mapping, or returns {@code null} if the mapper does not
+ * produce an element for this row (e.g. a null ID field). Reject AUTOMATIC-vertex mappings on
+ * update because the existing vertex cannot be identified.
+ */
+ static GraphElementEnvelope mapToEnvelope(
+ MappingEntry entry, SeaTunnelRow row, boolean update) {
+ if (update
+ && entry.config.getType() == LabelType.VERTEX
+ && entry.config.getIdStrategy()
+ == org.apache.hugegraph.structure.constant.IdStrategy.AUTOMATIC) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "Mapping[VERTEX/%s]: UPDATE_AFTER is not supported with AUTOMATIC IDs because the existing vertex cannot be identified",
+ entry.config.getLabel()));
+ }
+ GraphElement element;
+ try {
+ element = entry.mapper.map(row);
+ } catch (Exception e) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
+ String.format(
+ "Mapping[%s/%s]: Failed to map input row to graph element",
+ entry.config.getType(), entry.config.getLabel()),
+ e);
+ }
+ if (element == null) {
+ return null;
+ }
+ return new GraphElementEnvelope(
+ entry.config.getLabel(),
+ entry.config.getType(),
+ element,
+ entry.config.getUpdateStrategies());
+ }
+
+ /**
+ * INSERT/append-path mapping that supports unfold: returns one envelope normally, or N when a
+ * mapping expands a list-valued id cell into multiple elements.
+ */
+ static List<GraphElementEnvelope> mapToEnvelopes(MappingEntry entry, SeaTunnelRow row) {
+ List<GraphElement> elements;
+ try {
+ elements = entry.mapper.mapAll(row);
} catch (Exception e) {
throw new HugeGraphConnectorException(
HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
- "Non-retryable error executing graph operation",
+ String.format(
+ "Mapping[%s/%s]: Failed to map input row to graph element(s)",
+ entry.config.getType(), entry.config.getLabel()),
+ e);
+ }
+ if (elements == null || elements.isEmpty()) {
+ return Collections.emptyList();
+ }
+ List<GraphElementEnvelope> envelopes = new ArrayList<>(elements.size());
+ for (GraphElement element : elements) {
+ if (element == null) {
+ continue;
+ }
+ envelopes.add(
+ new GraphElementEnvelope(
+ entry.config.getLabel(),
+ entry.config.getType(),
+ element,
+ entry.config.getUpdateStrategies()));
+ }
+ return envelopes;
+ }
+
+ static final class UpdatePlan {
+ final List<GraphElementEnvelope> newVertices;
+ final List<GraphElementEnvelope> newEdges;
+ final List<Superseded> supersededVertices;
+ final List<Superseded> supersededEdges;
+
+ UpdatePlan(
+ List<GraphElementEnvelope> newVertices,
+ List<GraphElementEnvelope> newEdges,
+ List<Superseded> supersededVertices,
+ List<Superseded> supersededEdges) {
+ this.newVertices = newVertices;
+ this.newEdges = newEdges;
+ this.supersededVertices = supersededVertices;
+ this.supersededEdges = supersededEdges;
+ }
+ }
+
+ static final class Superseded {
+ final MappingEntry entry;
+ final Object oldId;
+
+ Superseded(MappingEntry entry, Object oldId) {
+ this.entry = entry;
+ this.oldId = oldId;
+ }
+ }
+
+ private void handleDelete(SeaTunnelRow row) {
+ rejectUnfoldForChangelog("DELETE");
+ rejectAutomaticVertexDelete(mappingEntries);
+
+ // Phase 1: build deletion plan — validate ALL mappings before touching the server.
+ // Any failure here (null id, missing field) aborts with zero side effects.
+ DeletePlan deletePlan = buildDeletePlan(row);
+
+ // Phase 2: execute the plan atomically — flush pending inserts first, then execute all
+ // deletions. The flush ensures earlier INSERT/UPSERT rows are persisted before their
+ // elements are deleted (a DELETE arriving immediately after its own INSERT).
+ try {
+ buffer.flush();
+ } catch (IOException e) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
+ "Failed to flush buffer before DELETE operation",
e);
}
+
+ executeDeletePlan(deletePlan);
+ }
+
+ /**
+ * Validates every mapping's ability to extract a delete ID from the row WITHOUT issuing any
+ * remote operation. Returns the validated plan — if this method returns, every ID is valid and
+ * the caller can safely execute them.
+ */
+ static DeletePlan buildDeletePlan(
+ List<MappingEntry> mappingEntries, SeaTunnelRow row, HugeGraphSinkConfig sinkConfig) {
+ List<DeleteTarget> edgeTargets = new ArrayList<>();
+ List<DeleteTarget> vertexTargets = new ArrayList<>();
+
+ for (MappingEntry entry : mappingEntries) {
+ Object id = entry.mapper.extractId(row);
+ if (id == null) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "Mapping[%s/%s]: Cannot delete because a required ID field is null "
+ + "or matches nullValues",
+ entry.config.getType(), entry.config.getLabel()));
+ }
+ if (entry.config.getType() == LabelType.VERTEX) {
+ vertexTargets.add(
+ new DeleteTarget(entry, id, sinkConfig.isDeleteVertexWithEdges()));
+ } else {
+ edgeTargets.add(new DeleteTarget(entry, id, false));
+ }
+ }
+
+ return new DeletePlan(edgeTargets, vertexTargets);
+ }
+
+ private DeletePlan buildDeletePlan(SeaTunnelRow row) {
+ return buildDeletePlan(mappingEntries, row, sinkConfig);
+ }
+
+ private void executeDeletePlan(DeletePlan plan) {
+ // Edges before vertices for topology safety.
+ for (DeleteTarget target : plan.edgeTargets) {
+ client.deleteEdge((String) target.id);
+ }
+ for (DeleteTarget target : plan.vertexTargets) {
+ if (target.deleteWithEdges) {
+ client.deleteVertexWithEdges(target.id);
+ } else {
+ client.deleteVertex(target.id);
+ }
+ }
+ }
+
+ static final class DeletePlan {
+ final List<DeleteTarget> edgeTargets;
+ final List<DeleteTarget> vertexTargets;
+
+ DeletePlan(List<DeleteTarget> edgeTargets, List<DeleteTarget> vertexTargets) {
+ this.edgeTargets = edgeTargets;
+ this.vertexTargets = vertexTargets;
+ }
+ }
+
+ static final class DeleteTarget {
+ final MappingEntry entry;
+ final Object id;
+ final boolean deleteWithEdges;
+
+ DeleteTarget(MappingEntry entry, Object id, boolean deleteWithEdges) {
+ this.entry = entry;
+ this.id = id;
+ this.deleteWithEdges = deleteWithEdges;
+ }
}
@Override
public Optional<Void> prepareCommit() {
+ // The SeaTunnel SinkWriter interface has no snapshotState() — the framework
+ // provides no mechanism for a sink writer to persist in-flight state across a
+ // checkpoint. A pending UPDATE_BEFORE means an update was split across the
+ // checkpoint boundary: UPDATE_BEFORE arrived but its paired UPDATE_AFTER has
+ // not yet been processed. On recovery the source replays from its own
+ // checkpoint, which may or may not include both rows, so the mutation could be
+ // partially applied (UPDATE_BEFORE replayed without its AFTER, or vice versa).
+ //
+ // Refuse to checkpoint in this state. Check BEFORE the flush try-catch so the
+ // exception propagates directly rather than being wrapped as a flush failure.
+ if (pendingUpdateBefore != null) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
+ "Checkpoint requested between UPDATE_BEFORE and UPDATE_AFTER: "
+ + "UPDATE_BEFORE received but its paired UPDATE_AFTER has "
+ + "not yet arrived. The SeaTunnel SinkWriter interface does "
+ + "not support persisting in-flight mutation state across "
+ + "checkpoints. UPDATE_BEFORE and UPDATE_AFTER must arrive "
+ + "within the same checkpoint interval. "
+ + "Mitigations: (1) increase the checkpoint interval, "
+ + "(2) use INSERT-only mode if the source does not emit "
+ + "changelog events.");
+ }
+
try {
buffer.flush();
- } catch (IOException e) {
+ } catch (Exception e) {
LOG.error("Failed to flush data during prepareCommit, failing checkpoint.", e);
throw new RuntimeException("Failed to flush data during prepareCommit()", e);
}
@@ -180,12 +792,49 @@ public class HugeGraphSinkWriter extends AbstractSinkWriter<SeaTunnelRow, Void>
@Override
public void close() throws IOException {
- if (buffer != null) {
- buffer.close();
+ Exception failure = null;
+ try {
+ if (buffer != null) {
+ buffer.close();
+ }
+ } catch (Exception e) {
+ failure = e;
+ } finally {
+ try {
+ if (client != null) {
+ client.close();
+ }
+ } catch (Exception closeFailure) {
+ if (failure == null) {
+ failure = closeFailure;
+ } else {
+ failure.addSuppressed(closeFailure);
+ }
+ }
+ }
+ if (failure instanceof IOException) {
+ throw (IOException) failure;
+ }
+ if (failure instanceof RuntimeException) {
+ throw (RuntimeException) failure;
}
+ if (failure != null) {
+ throw new IOException("Failed to close HugeGraph sink writer", failure);
+ }
+ }
+
+ /** Package-private test accessor. */
+ List<MappingEntry> mappingEntries() {
+ return mappingEntries;
+ }
+
+ static class MappingEntry {
+ final MappingConfig config;
+ final GraphDataMapper mapper;
- if (client != null) {
- client.close();
+ MappingEntry(MappingConfig config, GraphDataMapper mapper) {
+ this.config = config;
+ this.mapper = mapper;
}
}
}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/source/HugeGraphSource.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/source/HugeGraphSource.java
new file mode 100644
index 0000000000..80a681e2ff
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/source/HugeGraphSource.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.hugegraph.source;
+
+import org.apache.seatunnel.api.source.Boundedness;
+import org.apache.seatunnel.api.source.SeaTunnelSource;
+import org.apache.seatunnel.api.source.SourceReader;
+import org.apache.seatunnel.api.source.SourceSplitEnumerator;
+import org.apache.seatunnel.api.source.SupportColumnProjection;
+import org.apache.seatunnel.api.source.SupportParallelism;
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.HugeGraphOptions;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.HugeGraphSourceConfig;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * HugeGraph source. Bounded read of one vertex/edge label, or of all labels of a type in one job.
+ *
+ * <p>Single-label mode (option {@code label} set): at parallelism 1 it pages the label via the
+ * server-side list API (server-side label and property-equality filtering); at parallelism > 1
+ * it splits the keyspace into shards and scans them in parallel. Read-all mode ({@code label}
+ * omitted): one {@code LABEL_LIST} split per discovered label, each producing its own table. See
+ * {@link HugeGraphSourceSplitEnumerator}.
+ */
+public class HugeGraphSource
+ implements SeaTunnelSource<SeaTunnelRow, HugeGraphSourceSplit, HugeGraphSourceState>,
+ SupportParallelism,
+ SupportColumnProjection {
+
+ private static final long serialVersionUID = 1L;
+
+ private final List<CatalogTable> catalogTables;
+ private final Map<String, LabelTableContext> labelContexts;
+ private final HugeGraphSourceConfig sourceConfig;
+
+ public HugeGraphSource(
+ List<CatalogTable> catalogTables,
+ Map<String, LabelTableContext> labelContexts,
+ HugeGraphSourceConfig sourceConfig) {
+ this.catalogTables = catalogTables;
+ this.labelContexts = labelContexts;
+ this.sourceConfig = sourceConfig;
+ }
+
+ @Override
+ public String getPluginName() {
+ return HugeGraphOptions.PLUGIN_NAME;
+ }
+
+ @Override
+ public Boundedness getBoundedness() {
+ return Boundedness.BOUNDED;
+ }
+
+ @Override
+ public List<CatalogTable> getProducedCatalogTables() {
+ return catalogTables;
+ }
+
+ @Override
+ public SourceReader<SeaTunnelRow, HugeGraphSourceSplit> createReader(
+ SourceReader.Context readerContext) {
+ return new HugeGraphSourceReader(readerContext, sourceConfig, labelContexts);
+ }
+
+ @Override
+ public SourceSplitEnumerator<HugeGraphSourceSplit, HugeGraphSourceState> createEnumerator(
+ SourceSplitEnumerator.Context<HugeGraphSourceSplit> enumeratorContext) {
+ return new HugeGraphSourceSplitEnumerator(
+ enumeratorContext, sourceConfig, sourceConfig.getSplitSize());
+ }
+
+ @Override
+ public SourceSplitEnumerator<HugeGraphSourceSplit, HugeGraphSourceState> restoreEnumerator(
+ SourceSplitEnumerator.Context<HugeGraphSourceSplit> enumeratorContext,
+ HugeGraphSourceState checkpointState) {
+ return new HugeGraphSourceSplitEnumerator(
+ enumeratorContext, sourceConfig, sourceConfig.getSplitSize(), checkpointState);
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/source/HugeGraphSourceFactory.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/source/HugeGraphSourceFactory.java
new file mode 100644
index 0000000000..c916cea4b2
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/source/HugeGraphSourceFactory.java
@@ -0,0 +1,340 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.hugegraph.source;
+
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.api.configuration.util.OptionRule;
+import org.apache.seatunnel.api.options.ConnectorCommonOptions;
+import org.apache.seatunnel.api.options.EnvCommonOptions;
+import org.apache.seatunnel.api.source.SeaTunnelSource;
+import org.apache.seatunnel.api.source.SourceSplit;
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.catalog.CatalogTableUtil;
+import org.apache.seatunnel.api.table.connector.TableSource;
+import org.apache.seatunnel.api.table.factory.Factory;
+import org.apache.seatunnel.api.table.factory.TableSourceFactory;
+import org.apache.seatunnel.api.table.factory.TableSourceFactoryContext;
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.client.HugeGraphClient;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.client.HugeGraphOperations;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.HugeGraphConnectionConfig;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.HugeGraphOptions;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.HugeGraphSourceConfig;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.HugeGraphSourceOptions;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.MappingConfig;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.ReservedColumns;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorErrorCode;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorException;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.utils.HugeGraphTypeConverter;
+
+import com.google.auto.service.AutoService;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+@AutoService(Factory.class)
+public class HugeGraphSourceFactory implements TableSourceFactory {
+
+ @Override
+ public String factoryIdentifier() {
+ return HugeGraphOptions.PLUGIN_NAME;
+ }
+
+ @Override
+ public OptionRule optionRule() {
+ return OptionRule.builder()
+ .required(HugeGraphOptions.HOST, HugeGraphOptions.PORT, HugeGraphOptions.GRAPH_NAME)
+ .optional(
+ // When omitted, the source reads ALL labels of label_type (default VERTEX),
+ // producing one table per label. When set, only that one label is read.
+ HugeGraphSourceOptions.LABEL,
+ // Optional: when omitted, the property columns are auto-discovered from the
+ // server label definition (all properties, inferred types).
+ ConnectorCommonOptions.SCHEMA,
+ HugeGraphSourceOptions.LABEL_TYPE,
+ HugeGraphSourceOptions.PAGE_SIZE,
+ HugeGraphSourceOptions.SPLIT_SIZE,
+ HugeGraphSourceOptions.FILTER,
+ HugeGraphSourceOptions.TIME_ZONE,
+ HugeGraphOptions.PROTOCOL,
+ HugeGraphOptions.USERNAME,
+ HugeGraphOptions.PASSWORD,
+ // Optional connection setting passed through to select the HugeGraph graph
+ // space (defaults to "DEFAULT").
+ HugeGraphOptions.GRAPH_SPACE,
+ HugeGraphOptions.MAX_RETRIES,
+ HugeGraphOptions.RETRY_BACKOFF_MS,
+ HugeGraphOptions.RETRY_BACKOFF_MAX_MS)
+ .build();
+ }
+
+ @Override
+ public Class<? extends SeaTunnelSource> getSourceClass() {
+ return HugeGraphSource.class;
+ }
+
+ @Override
+ public <T, SplitT extends SourceSplit, StateT extends Serializable>
+ TableSource<T, SplitT, StateT> createSource(TableSourceFactoryContext context) {
+ ReadonlyConfig options = context.getOptions();
+ MappingConfig.LabelType labelType =
+ options.getOptional(HugeGraphSourceOptions.LABEL_TYPE)
+ .orElse(HugeGraphSourceOptions.LABEL_TYPE.defaultValue());
+ boolean readAll = !options.getOptional(HugeGraphSourceOptions.LABEL).isPresent();
+
+ List<CatalogTable> catalogTables = new ArrayList<>();
+ Map<String, LabelTableContext> labelContexts = new LinkedHashMap<>();
+ HugeGraphSourceConfig sourceConfig;
+
+ if (readAll) {
+ sourceConfig = buildReadAllTables(options, labelType, catalogTables, labelContexts);
+ } else {
+ checkFilterParallelism(options);
+ CatalogTable propertyCatalogTable = resolvePropertyCatalogTable(options, labelType);
+ SeaTunnelRowType propertyRowType = propertyCatalogTable.getSeaTunnelRowType();
+ sourceConfig = HugeGraphSourceConfig.of(options, propertyRowType);
+ CatalogTable producedCatalogTable =
+ CatalogTableUtil.newCatalogTable(
+ propertyCatalogTable,
+ prependReservedFields(propertyRowType, sourceConfig.getLabelType()));
+ catalogTables.add(producedCatalogTable);
+ labelContexts.put(
+ sourceConfig.getLabel(),
+ new LabelTableContext(
+ sourceConfig.getLabel(),
+ propertyRowType,
+ producedCatalogTable.getSeaTunnelRowType(),
+ producedCatalogTable.getTablePath().toString()));
+ }
+
+ final List<CatalogTable> tables = catalogTables;
+ final Map<String, LabelTableContext> contexts = labelContexts;
+ final HugeGraphSourceConfig cfg = sourceConfig;
+ return () ->
+ (SeaTunnelSource<T, SplitT, StateT>) new HugeGraphSource(tables, contexts, cfg);
+ }
+
+ /**
+ * Read-all mode: discover every label of {@code labelType} from the server and build one
+ * produced {@link CatalogTable} + one {@link LabelTableContext} per label. Rejects {@code
+ * schema}/{@code filter} (neither can describe multiple heterogeneous labels) and an empty
+ * graph up front. Returns the read-all {@link HugeGraphSourceConfig}.
+ */
+ private HugeGraphSourceConfig buildReadAllTables(
+ ReadonlyConfig options,
+ MappingConfig.LabelType labelType,
+ List<CatalogTable> catalogTables,
+ Map<String, LabelTableContext> labelContexts) {
+ if (options.getOptional(ConnectorCommonOptions.SCHEMA).isPresent()) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ "'schema' cannot be combined with reading all labels (option 'label' omitted): "
+ + "a single schema cannot describe multiple labels. Set 'label' to use "
+ + "'schema', or drop 'schema' to auto-discover every label.");
+ }
+ boolean hasFilter =
+ options.getOptional(HugeGraphSourceOptions.FILTER)
+ .map(filter -> !filter.isEmpty())
+ .orElse(false);
+ if (hasFilter) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ "'filter' cannot be combined with reading all labels (option 'label' omitted): "
+ + "a property-equality filter assumes the property exists on every "
+ + "label. Set 'label' to use 'filter'.");
+ }
+ HugeGraphClient client = new HugeGraphClient(HugeGraphConnectionConfig.of(options));
+ List<String> labels;
+ try {
+ labels =
+ labelType == MappingConfig.LabelType.VERTEX
+ ? client.listVertexLabels()
+ : client.listEdgeLabels();
+ if (labels.isEmpty()) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "No %s labels found in HugeGraph graph '%s'; nothing to read.",
+ labelType == MappingConfig.LabelType.VERTEX ? "vertex" : "edge",
+ options.get(HugeGraphOptions.GRAPH_NAME)));
+ }
+ for (String label : labels) {
+ SeaTunnelRowType propertyRowType =
+ discoverPropertyRowType(client, label, labelType);
+ CatalogTable propertyTable =
+ CatalogTableUtil.getCatalogTable(label, propertyRowType);
+ CatalogTable producedTable =
+ CatalogTableUtil.newCatalogTable(
+ propertyTable, prependReservedFields(propertyRowType, labelType));
+ catalogTables.add(producedTable);
+ labelContexts.put(
+ label,
+ new LabelTableContext(
+ label,
+ propertyRowType,
+ producedTable.getSeaTunnelRowType(),
+ producedTable.getTablePath().toString()));
+ }
+ } finally {
+ client.close();
+ }
+ return HugeGraphSourceConfig.ofReadAll(options, labels);
+ }
+
+ /**
+ * Resolves the property columns. When {@code schema} is configured it is used verbatim;
+ * otherwise the columns are auto-discovered from the server label definition (all property
+ * keys, with types inferred from the server).
+ */
+ private CatalogTable resolvePropertyCatalogTable(
+ ReadonlyConfig options, MappingConfig.LabelType labelType) {
+ if (options.getOptional(ConnectorCommonOptions.SCHEMA).isPresent()) {
+ return CatalogTableUtil.buildWithConfig(options);
+ }
+ String label = options.get(HugeGraphSourceOptions.LABEL);
+ HugeGraphClient client = new HugeGraphClient(HugeGraphConnectionConfig.of(options));
+ SeaTunnelRowType propertyRowType;
+ try {
+ propertyRowType = discoverPropertyRowType(client, label, labelType);
+ } finally {
+ client.close();
+ }
+ return CatalogTableUtil.getCatalogTable(label, propertyRowType);
+ }
+
+ /**
+ * Builds the property row type from the server label definition: every property key of the
+ * label, ordered by name for a deterministic column order, typed via {@link
+ * HugeGraphTypeConverter}.
+ */
+ static SeaTunnelRowType discoverPropertyRowType(
+ HugeGraphOperations client, String label, MappingConfig.LabelType labelType) {
+ Set<String> properties =
+ labelType == MappingConfig.LabelType.VERTEX
+ ? client.getVertexLabelPropertiesOrNull(label)
+ : client.getEdgeLabelPropertiesOrNull(label);
+ if (properties == null) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "%s label '%s' does not exist in HugeGraph schema; cannot auto-discover "
+ + "its columns. Create the label or declare 'schema.fields'.",
+ labelType == MappingConfig.LabelType.VERTEX ? "Vertex" : "Edge",
+ label));
+ }
+ List<String> names = new ArrayList<>(properties);
+ Collections.sort(names);
+ String[] fieldNames = new String[names.size()];
+ SeaTunnelDataType<?>[] fieldTypes = new SeaTunnelDataType<?>[names.size()];
+ for (int i = 0; i < names.size(); i++) {
+ String name = names.get(i);
+ fieldNames[i] = name;
+ fieldTypes[i] =
+ HugeGraphTypeConverter.toSeaTunnelType(
+ client.getPropertyDataType(name),
+ client.getPropertyCardinality(name),
+ name);
+ }
+ return new SeaTunnelRowType(fieldNames, fieldTypes);
+ }
+
+ /**
+ * Parallelism > 1 uses shard-based key-range scans, which cannot push a property-equality
+ * {@code filter} to the server (the scan API takes no condition). Reject that combination here,
+ * at plan/config time, so the user gets an actionable choice before the job starts rather than
+ * a silently-ignored filter or a mid-run failure. {@code filter} with parallelism = 1
+ * (label-list scan) is fully supported.
+ */
+ static void checkFilterParallelism(ReadonlyConfig options) {
+ int parallelism = options.getOptional(EnvCommonOptions.PARALLELISM).orElse(1);
+ boolean hasFilter =
+ options.getOptional(HugeGraphSourceOptions.FILTER)
+ .map(filter -> !filter.isEmpty())
+ .orElse(false);
+ if (parallelism > 1 && hasFilter) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "HugeGraph source 'filter' cannot be combined with parallelism > 1 "
+ + "(got %d): parallel reads use shard key-range scans that do not "
+ + "support server-side property filtering. Either set parallelism "
+ + "to 1 to keep the filter, or remove the filter to read in "
+ + "parallel.",
+ parallelism));
+ }
+ }
+
+ static SeaTunnelRowType prependReservedFields(
+ SeaTunnelRowType propertyRowType, MappingConfig.LabelType labelType) {
+ // The source auto-prepends reserved columns (~id/~label, plus edge endpoints). A user
+ // schema.fields column with a reserved name would silently create a duplicate column and
+ // later fail with a misleading "label has no property ~id"; reject it up front with a clear
+ // message instead. Auto-discovered names never start with '~' (HugeGraph forbids it), so
+ // this only triggers on an explicit schema.fields declaration.
+ for (String fieldName : propertyRowType.getFieldNames()) {
+ if (ReservedColumns.isReserved(fieldName)) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "schema.fields must not declare the reserved column '%s': names "
+ + "starting with '%s' are emitted automatically by the "
+ + "HugeGraph source (%s for vertices; also %s, %s, %s, %s "
+ + "for edges). Remove '%s' from schema.fields.",
+ fieldName,
+ ReservedColumns.PREFIX,
+ ReservedColumns.ID + "/" + ReservedColumns.LABEL,
+ ReservedColumns.SOURCE_ID,
+ ReservedColumns.SOURCE_LABEL,
+ ReservedColumns.TARGET_ID,
+ ReservedColumns.TARGET_LABEL,
+ fieldName));
+ }
+ }
+ int reservedSize = labelType == MappingConfig.LabelType.VERTEX ? 2 : 6;
+ String[] fieldNames = new String[reservedSize + propertyRowType.getTotalFields()];
+ SeaTunnelDataType<?>[] fieldTypes =
+ new SeaTunnelDataType<?>[reservedSize + propertyRowType.getTotalFields()];
+
+ fieldNames[0] = HugeGraphSourceReader.ID_FIELD;
+ fieldNames[1] = HugeGraphSourceReader.LABEL_FIELD;
+ fieldTypes[0] = BasicType.STRING_TYPE;
+ fieldTypes[1] = BasicType.STRING_TYPE;
+ if (labelType == MappingConfig.LabelType.EDGE) {
+ fieldNames[2] = HugeGraphSourceReader.SOURCE_ID_FIELD;
+ fieldNames[3] = HugeGraphSourceReader.SOURCE_LABEL_FIELD;
+ fieldNames[4] = HugeGraphSourceReader.TARGET_ID_FIELD;
+ fieldNames[5] = HugeGraphSourceReader.TARGET_LABEL_FIELD;
+ for (int i = 2; i < reservedSize; i++) {
+ fieldTypes[i] = BasicType.STRING_TYPE;
+ }
+ }
+
+ for (int i = 0; i < propertyRowType.getTotalFields(); i++) {
+ fieldNames[reservedSize + i] = propertyRowType.getFieldName(i);
+ fieldTypes[reservedSize + i] = propertyRowType.getFieldType(i);
+ }
+ return new SeaTunnelRowType(fieldNames, fieldTypes);
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/source/HugeGraphSourceReader.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/source/HugeGraphSourceReader.java
new file mode 100644
index 0000000000..6fab5cdc16
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/source/HugeGraphSourceReader.java
@@ -0,0 +1,669 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.hugegraph.source;
+
+import org.apache.seatunnel.api.source.Boundedness;
+import org.apache.seatunnel.api.source.Collector;
+import org.apache.seatunnel.api.source.SourceReader;
+import org.apache.seatunnel.api.table.type.ArrayType;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRow;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.api.table.type.SqlType;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.client.HugeGraphClient;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.client.HugeGraphOperations;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.client.PageResult;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.HugeGraphSourceConfig;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.MappingConfig;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorErrorCode;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorException;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.utils.HugeGraphTypeConverter;
+
+import org.apache.hugegraph.structure.constant.Cardinality;
+import org.apache.hugegraph.structure.constant.DataType;
+import org.apache.hugegraph.structure.graph.Edge;
+import org.apache.hugegraph.structure.graph.Vertex;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.lang.reflect.Array;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.DateTimeParseException;
+import java.time.temporal.ChronoField;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Deque;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentLinkedDeque;
+
+/**
+ * Reads HugeGraph vertices/edges into SeaTunnel rows, one assigned split at a time.
+ *
+ * <p>A {@code LABEL_LIST} split pages the whole label via the server-side list API (with optional
+ * server-side property filtering). A {@code SHARD} split scans a key range via the scan API and
+ * filters to the configured label client-side (the scan API cannot filter by label or property).
+ * Progress (page marker, finished flag, dedup id) is stored on the split so a checkpoint can resume
+ * mid-scan after failover.
+ */
+public class HugeGraphSourceReader implements SourceReader<SeaTunnelRow, HugeGraphSourceSplit> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HugeGraphSourceReader.class);
+
+ private static final long IDLE_POLL_INTERVAL_MS = 1000L;
+
+ /** HugeGraph server DATE serialization format: {@code yyyy-MM-dd HH:mm:ss.SSS}. */
+ private static final DateTimeFormatter HUGEGRAPH_DATE_FORMAT =
+ new DateTimeFormatterBuilder()
+ .appendPattern("yyyy-MM-dd HH:mm:ss")
+ .appendFraction(ChronoField.MILLI_OF_SECOND, 0, 3, true)
+ .toFormatter();
+
+ public static final String ID_FIELD = "~id";
+ public static final String LABEL_FIELD = "~label";
+ public static final String SOURCE_ID_FIELD = "~source_id";
+ public static final String SOURCE_LABEL_FIELD = "~source_label";
+ public static final String TARGET_ID_FIELD = "~target_id";
+ public static final String TARGET_LABEL_FIELD = "~target_label";
+
+ private final SourceReader.Context context;
+ private final HugeGraphSourceConfig sourceConfig;
+ // Per-label read context, keyed by label. Single-label mode has exactly one entry; read-all
+ // mode has one per discovered label. The reader resolves the entry by each split's active
+ // label.
+ private final Map<String, LabelTableContext> labelContexts;
+ private final HugeGraphOperations client;
+
+ private final Deque<HugeGraphSourceSplit> pendingSplits = new ConcurrentLinkedDeque<>();
+ private HugeGraphSourceSplit currentSplit;
+ private volatile boolean noMoreSplits;
+
+ // Dedup state for the split currently being read; mirrored to/from currentSplit around each
+ // page
+ // so it survives checkpoints.
+ private String lastEmittedId;
+ private long duplicateSkipped;
+ private long totalRecords;
+ private int pageCount;
+
+ public HugeGraphSourceReader(
+ SourceReader.Context context,
+ HugeGraphSourceConfig sourceConfig,
+ Map<String, LabelTableContext> labelContexts) {
+ this(
+ context,
+ sourceConfig,
+ labelContexts,
+ new HugeGraphClient(sourceConfig.getConnectionConfig()));
+ }
+
+ HugeGraphSourceReader(
+ SourceReader.Context context,
+ HugeGraphSourceConfig sourceConfig,
+ Map<String, LabelTableContext> labelContexts,
+ HugeGraphOperations client) {
+ this.context = context;
+ this.sourceConfig = sourceConfig;
+ this.labelContexts = labelContexts;
+ this.client = client;
+ }
+
+ /**
+ * The label the given split reads: a LABEL_LIST split names it directly; a SHARD split scans a
+ * key range of all labels and inherits the single configured label (shard splits are only
+ * created in single-label mode).
+ */
+ private String activeLabel(HugeGraphSourceSplit split) {
+ return split.getLabel() != null ? split.getLabel() : sourceConfig.getLabel();
+ }
+
+ private LabelTableContext contextFor(String label) {
+ LabelTableContext ctx = labelContexts.get(label);
+ if (ctx == null) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format("No read context for label '%s'.", label));
+ }
+ return ctx;
+ }
+
+ @Override
+ public void open() {
+ // Read-all mode auto-discovers each label's row type from the server, so there is no user
+ // schema to validate against (it cannot mismatch). Skip validation; the reader resolves
+ // each split's context by label at read time.
+ if (sourceConfig.isReadAllLabels()) {
+ return;
+ }
+ try {
+ // validateLabelAndSchema triggers the lazy client connection; if it throws (unknown
+ // label, type mismatch, unreachable server) the framework may not call close(), so
+ // release the just-opened client here to avoid leaking connection pools/threads.
+ validateLabelAndSchema();
+ } catch (RuntimeException e) {
+ try {
+ client.close();
+ } catch (RuntimeException closeFailure) {
+ e.addSuppressed(closeFailure);
+ }
+ throw e;
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ client.close();
+ }
+
+ @Override
+ public void addSplits(List<HugeGraphSourceSplit> splits) {
+ if (splits != null) {
+ pendingSplits.addAll(splits);
+ }
+ }
+
+ @Override
+ public void handleNoMoreSplits() {
+ noMoreSplits = true;
+ }
+
+ @Override
+ public List<HugeGraphSourceSplit> snapshotState(long checkpointId) {
+ List<HugeGraphSourceSplit> state = new ArrayList<>();
+ HugeGraphSourceSplit cur = currentSplit;
+ if (cur != null && !cur.isFinished()) {
+ state.add(cur);
+ }
+ state.addAll(pendingSplits);
+ return state;
+ }
+
+ @Override
+ public void notifyCheckpointComplete(long checkpointId) {
+ // No-op: the source keeps no server-side cursor to acknowledge.
+ }
+
+ @Override
+ public void pollNext(Collector<SeaTunnelRow> output) throws InterruptedException {
+ boolean idle = false;
+ synchronized (output.getCheckpointLock()) {
+ if (currentSplit == null) {
+ currentSplit = pendingSplits.poll();
+ }
+ if (currentSplit == null) {
+ if (noMoreSplits && Boundedness.BOUNDED.equals(context.getBoundedness())) {
+ context.signalNoMoreElement();
+ } else {
+ context.sendSplitRequest();
+ idle = true;
+ }
+ } else {
+ readOnePage(currentSplit, output);
+ }
+ }
+ if (idle) {
+ Thread.sleep(IDLE_POLL_INTERVAL_MS);
+ }
+ }
+
+ /** Reads one bounded page of the current split so checkpoints can persist progress. */
+ private void readOnePage(HugeGraphSourceSplit split, Collector<SeaTunnelRow> output) {
+ String requestedPage = split.getPage();
+ this.lastEmittedId = split.getLastEmittedId();
+ String label = activeLabel(split);
+ LabelTableContext ctx = contextFor(label);
+
+ int recordCount;
+ String responsePage;
+ if (sourceConfig.getLabelType() == MappingConfig.LabelType.VERTEX) {
+ PageResult<Vertex> page = fetchVertexPage(split, label, requestedPage);
+ List<Vertex> records =
+ split.isShardMode()
+ ? filterVerticesByLabel(page.getRecords(), label)
+ : page.getRecords();
+ collectVertices(records, output, ctx);
+ recordCount = page.getRecords().size();
+ responsePage = page.getNextPage();
+ } else {
+ PageResult<Edge> page = fetchEdgePage(split, label, requestedPage);
+ List<Edge> records =
+ split.isShardMode()
+ ? filterEdgesByLabel(page.getRecords(), label)
+ : page.getRecords();
+ collectEdges(records, output, ctx);
+ recordCount = page.getRecords().size();
+ responsePage = page.getNextPage();
+ }
+
+ if (responsePage != null && responsePage.equals(requestedPage)) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
+ String.format(
+ "HugeGraph pagination marker did not advance for split '%s': '%s'",
+ split.splitId(), responsePage));
+ }
+
+ split.setLastEmittedId(this.lastEmittedId);
+ split.setPage(responsePage);
+ totalRecords += recordCount;
+ pageCount++;
+ if (recordCount == 0 && responsePage != null) {
+ LOG.debug(
+ "HugeGraph source received an empty intermediate page for split '{}'; continuing",
+ split.splitId());
+ }
+ if (responsePage == null) {
+ split.setFinished(true);
+ LOG.info(
+ "HugeGraph source finished split '{}' (label '{}'): {} records in {} pages"
+ + " ({} server-side paging duplicates skipped)",
+ split.splitId(),
+ label,
+ totalRecords,
+ pageCount,
+ duplicateSkipped);
+ // Reset per-split counters for the next split.
+ totalRecords = 0;
+ pageCount = 0;
+ duplicateSkipped = 0;
+ currentSplit = null;
+ }
+ }
+
+ private PageResult<Vertex> fetchVertexPage(
+ HugeGraphSourceSplit split, String label, String requestedPage) {
+ if (split.isShardMode()) {
+ return client.scanVertices(split.toShard(), requestedPage, sourceConfig.getPageSize());
+ }
+ return client.listVertices(
+ label, sourceConfig.getFilter(), requestedPage, sourceConfig.getPageSize());
+ }
+
+ private PageResult<Edge> fetchEdgePage(
+ HugeGraphSourceSplit split, String label, String requestedPage) {
+ if (split.isShardMode()) {
+ return client.scanEdges(split.toShard(), requestedPage, sourceConfig.getPageSize());
+ }
+ return client.listEdges(
+ label, sourceConfig.getFilter(), requestedPage, sourceConfig.getPageSize());
+ }
+
+ /**
+ * Shard scans return elements of all labels in the key range; keep only {@code label}. (Shard
+ * mode is single-label only, so this filters to the one configured label.)
+ */
+ private List<Vertex> filterVerticesByLabel(List<Vertex> records, String label) {
+ List<Vertex> filtered = new ArrayList<>(records.size());
+ for (Vertex vertex : records) {
+ if (label.equals(vertex.label())) {
+ filtered.add(vertex);
+ }
+ }
+ return filtered;
+ }
+
+ private List<Edge> filterEdgesByLabel(List<Edge> records, String label) {
+ List<Edge> filtered = new ArrayList<>(records.size());
+ for (Edge edge : records) {
+ if (label.equals(edge.label())) {
+ filtered.add(edge);
+ }
+ }
+ return filtered;
+ }
+
+ private void validateLabelAndSchema() {
+ // Single-label mode only (read-all skips validation in open()); exactly one context, keyed
+ // by the configured label.
+ SeaTunnelRowType propertyRowType = contextFor(sourceConfig.getLabel()).getPropertyRowType();
+ Set<String> labelProperties;
+ if (sourceConfig.getLabelType() == MappingConfig.LabelType.VERTEX) {
+ labelProperties = client.getVertexLabelPropertiesOrNull(sourceConfig.getLabel());
+ if (labelProperties == null) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Vertex label '%s' does not exist in HugeGraph schema",
+ sourceConfig.getLabel()));
+ }
+ } else {
+ labelProperties = client.getEdgeLabelPropertiesOrNull(sourceConfig.getLabel());
+ if (labelProperties == null) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Edge label '%s' does not exist in HugeGraph schema",
+ sourceConfig.getLabel()));
+ }
+ }
+
+ for (int i = 0; i < propertyRowType.getTotalFields(); i++) {
+ String propertyName = propertyRowType.getFieldName(i);
+ if (!labelProperties.contains(propertyName)) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Label '%s' does not contain property '%s'. Available properties: %s",
+ sourceConfig.getLabel(), propertyName, labelProperties));
+ }
+ validatePropertyType(propertyName, propertyRowType.getFieldType(i));
+ }
+
+ // A filter keyed on a non-existent property would be silently dropped by the server and
+ // return the whole label — fail fast so the misconfiguration surfaces at open() instead.
+ // Values are also coerced to the property's server type: the server matches by typed value,
+ // so a BOOLEAN property filtered with the string "true" (or a LONG filtered with an int)
+ // would otherwise match nothing and return 0 rows with no error.
+ Map<String, Object> filter = sourceConfig.getFilter();
+ if (filter != null && !filter.isEmpty()) {
+ Map<String, Object> coerced = new LinkedHashMap<>();
+ for (Map.Entry<String, Object> entry : filter.entrySet()) {
+ String key = entry.getKey();
+ if (!labelProperties.contains(key)) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "filter property '%s' is not a property of label '%s'. "
+ + "Available properties: %s",
+ key, sourceConfig.getLabel(), labelProperties));
+ }
+ coerced.put(
+ key,
+ coerceFilterValue(key, entry.getValue(), client.getPropertyDataType(key)));
+ }
+ sourceConfig.setFilter(coerced);
+ }
+ }
+
+ /**
+ * Coerces a filter value to the Java type the HugeGraph server matches against for the
+ * property's type — config often supplies a string or a loosely-typed number. A value that
+ * cannot be coerced (e.g. {@code "yes"} for a BOOLEAN) fails fast here instead of silently
+ * matching nothing. DATE/BLOB/OBJECT are passed through unchanged (not sensibly filterable).
+ */
+ static Object coerceFilterValue(String key, Object value, DataType dataType) {
+ if (value == null) {
+ return null;
+ }
+ String raw = value.toString().trim();
+ try {
+ switch (dataType) {
+ case BOOLEAN:
+ if (value instanceof Boolean) {
+ return value;
+ }
+ if ("true".equalsIgnoreCase(raw)) {
+ return Boolean.TRUE;
+ }
+ if ("false".equalsIgnoreCase(raw)) {
+ return Boolean.FALSE;
+ }
+ throw new IllegalArgumentException("expected true or false");
+ case BYTE:
+ return value instanceof Number
+ ? ((Number) value).byteValue()
+ : Byte.valueOf(raw);
+ case INT:
+ return value instanceof Number
+ ? ((Number) value).intValue()
+ : Integer.valueOf(raw);
+ case LONG:
+ return value instanceof Number
+ ? ((Number) value).longValue()
+ : Long.valueOf(raw);
+ case FLOAT:
+ return value instanceof Number
+ ? ((Number) value).floatValue()
+ : Float.valueOf(raw);
+ case DOUBLE:
+ return value instanceof Number
+ ? ((Number) value).doubleValue()
+ : Double.valueOf(raw);
+ case TEXT:
+ case UUID:
+ return raw;
+ default:
+ return value;
+ }
+ } catch (RuntimeException e) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "filter value '%s' for property '%s' is not a valid %s.",
+ value, key, dataType),
+ e);
+ }
+ }
+
+ private void validatePropertyType(String propertyName, SeaTunnelDataType<?> seaTunnelType) {
+ Cardinality cardinality = client.getPropertyCardinality(propertyName);
+ DataType propertyDataType = client.getPropertyDataType(propertyName);
+ boolean serverIsMulti = cardinality != null && cardinality != Cardinality.SINGLE;
+ boolean declaredArray = seaTunnelType.getSqlType() == SqlType.ARRAY;
+
+ if (serverIsMulti && !declaredArray) {
+ // Guides the user to the fix; without this hint, the server's Collection value would
+ // ClassCastException mid-scan against the scalar row builder.
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Property '%s' has cardinality %s on the server but schema.fields "
+ + "declares it as '%s'. Declare it as 'array<%s>' to read the "
+ + "collection, or remove it from schema.fields.",
+ propertyName,
+ cardinality,
+ seaTunnelType,
+ toSeaTunnelType(propertyDataType, Cardinality.SINGLE, propertyName)));
+ }
+ if (declaredArray && !serverIsMulti) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Property '%s' is declared as ARRAY in schema.fields but has "
+ + "cardinality SINGLE on the server (type %s).",
+ propertyName, propertyDataType));
+ }
+ SeaTunnelDataType<?> expectedType =
+ toSeaTunnelType(propertyDataType, cardinality, propertyName);
+ if (!expectedType.equals(seaTunnelType)) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Type mismatch for property '%s': schema.fields declares '%s', "
+ + "but HugeGraph type '%s' (cardinality=%s) maps to '%s'",
+ propertyName,
+ seaTunnelType,
+ propertyDataType,
+ cardinality,
+ expectedType));
+ }
+ }
+
+ private SeaTunnelDataType<?> toSeaTunnelType(
+ DataType dataType, Cardinality cardinality, String propertyName) {
+ return HugeGraphTypeConverter.toSeaTunnelType(dataType, cardinality, propertyName);
+ }
+
+ private void collectVertices(
+ List<Vertex> vertices, Collector<SeaTunnelRow> output, LabelTableContext ctx) {
+ SeaTunnelRowType outputRowType = ctx.getOutputRowType();
+ for (Vertex vertex : vertices) {
+ String id = String.valueOf(vertex.id());
+ if (isAdjacentDuplicate(id)) {
+ continue;
+ }
+ Object[] fields = new Object[outputRowType.getTotalFields()];
+ fields[0] = id;
+ fields[1] = vertex.label();
+ fillProperties(vertex.properties(), fields, 2, ctx.getPropertyRowType());
+ SeaTunnelRow row = new SeaTunnelRow(fields);
+ row.setTableId(ctx.getTableId());
+ output.collect(row);
+ }
+ }
+
+ private void collectEdges(
+ List<Edge> edges, Collector<SeaTunnelRow> output, LabelTableContext ctx) {
+ SeaTunnelRowType outputRowType = ctx.getOutputRowType();
+ for (Edge edge : edges) {
+ String id = String.valueOf(edge.id());
+ if (isAdjacentDuplicate(id)) {
+ continue;
+ }
+ Object[] fields = new Object[outputRowType.getTotalFields()];
+ fields[0] = id;
+ fields[1] = edge.label();
+ fields[2] = String.valueOf(edge.sourceId());
+ fields[3] = edge.sourceLabel();
+ fields[4] = String.valueOf(edge.targetId());
+ fields[5] = edge.targetLabel();
+ fillProperties(edge.properties(), fields, 6, ctx.getPropertyRowType());
+ SeaTunnelRow row = new SeaTunnelRow(fields);
+ row.setTableId(ctx.getTableId());
+ output.collect(row);
+ }
+ }
+
+ /**
+ * The HugeGraph RocksDB backend emits one duplicate record at every internal 500-record scan
+ * boundary when limit >= 1000 (observed 2001 duplicates per 1M rows, all back-to-back).
+ * Element IDs are unique within a label, so two consecutive identical IDs can only be that
+ * server-side paging artifact — skip them with O(1) memory.
+ */
+ private boolean isAdjacentDuplicate(String id) {
+ if (id.equals(lastEmittedId)) {
+ duplicateSkipped++;
+ return true;
+ }
+ lastEmittedId = id;
+ return false;
+ }
+
+ private void fillProperties(
+ Map<String, Object> properties,
+ Object[] fields,
+ int propertyOffset,
+ SeaTunnelRowType propertyRowType) {
+ for (int i = 0; i < propertyRowType.getTotalFields(); i++) {
+ String propertyName = propertyRowType.getFieldName(i);
+ fields[propertyOffset + i] =
+ convertPropertyValue(
+ properties.get(propertyName), propertyRowType.getFieldType(i));
+ }
+ }
+
+ private Object convertPropertyValue(Object value, SeaTunnelDataType<?> targetType) {
+ if (value == null) {
+ return null;
+ }
+ switch (targetType.getSqlType()) {
+ case ARRAY:
+ return convertArrayValue(value, (ArrayType<?, ?>) targetType);
+ case TINYINT:
+ return ((Number) value).byteValue();
+ case INT:
+ return ((Number) value).intValue();
+ case BIGINT:
+ return ((Number) value).longValue();
+ case FLOAT:
+ return ((Number) value).floatValue();
+ case DOUBLE:
+ return ((Number) value).doubleValue();
+ case BOOLEAN:
+ if (value instanceof Boolean) {
+ return value;
+ }
+ return Boolean.parseBoolean(value.toString());
+ case BYTES:
+ if (value instanceof byte[]) {
+ return value;
+ }
+ return Base64.getDecoder().decode(value.toString());
+ case TIMESTAMP:
+ if (value instanceof Date) {
+ return LocalDateTime.ofInstant(((Date) value).toInstant(), getSourceZoneId());
+ }
+ if (value instanceof Number) {
+ return LocalDateTime.ofInstant(
+ new Date(((Number) value).longValue()).toInstant(), getSourceZoneId());
+ }
+ // A server-serialized wall-clock string carries no zone, so time_zone cannot be
+ // applied here without knowing the server's serialization zone — keep the value
+ // verbatim (documented on the time_zone option). time_zone applies only to the
+ // epoch/Date branches above.
+ return parseDateTime(value.toString());
+ case STRING:
+ return value.toString();
+ default:
+ return value;
+ }
+ }
+
+ /**
+ * Converts a HugeGraph LIST/SET property value (returned as a Collection by the client) into a
+ * typed SeaTunnel array. HugeGraph 1.5.0 returns LIST as {@code ArrayList} and SET as {@code
+ * HashSet}; both flow through {@link Collection} here. SET's original insertion order is not
+ * guaranteed by the server, so callers relying on stable ordering must use LIST.
+ */
+ private Object convertArrayValue(Object value, ArrayType<?, ?> targetType) {
+ if (!(value instanceof Collection)) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
+ String.format(
+ "Expected a Collection for ARRAY property, got %s",
+ value.getClass().getName()));
+ }
+ Collection<?> collection = (Collection<?>) value;
+ SeaTunnelDataType<?> elementType = targetType.getElementType();
+ Object array = Array.newInstance(elementType.getTypeClass(), collection.size());
+ int i = 0;
+ for (Object element : collection) {
+ Array.set(array, i++, convertPropertyValue(element, elementType));
+ }
+ return array;
+ }
+
+ private ZoneId getSourceZoneId() {
+ return sourceConfig.getTimeZone() == null
+ ? ZoneId.systemDefault()
+ : ZoneId.of(sourceConfig.getTimeZone());
+ }
+
+ /**
+ * Parses a HugeGraph DATE property returned as a String. The server serializes dates as {@code
+ * yyyy-MM-dd HH:mm:ss.SSS} (space separator, optional fractional seconds), which {@link
+ * LocalDateTime#parse} rejects because it only accepts the ISO 'T' separator. Accept both the
+ * space-separated server format and ISO-8601 (in case a future/config variant emits a 'T').
+ */
+ private static LocalDateTime parseDateTime(String text) {
+ try {
+ return LocalDateTime.parse(text, HUGEGRAPH_DATE_FORMAT);
+ } catch (DateTimeParseException e) {
+ return LocalDateTime.parse(text);
+ }
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/source/HugeGraphSourceSplit.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/source/HugeGraphSourceSplit.java
new file mode 100644
index 0000000000..957099ca82
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/source/HugeGraphSourceSplit.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.hugegraph.source;
+
+import org.apache.seatunnel.api.source.SourceSplit;
+
+import org.apache.hugegraph.structure.graph.Shard;
+
+import java.util.Objects;
+
+/**
+ * A unit of read work for the HugeGraph source.
+ *
+ * <p>Two modes:
+ *
+ * <ul>
+ * <li>{@code LABEL_LIST}: a single split that pages the whole label via the server-side list API
+ * ({@code /vertices?label=&page=}). Used when parallelism == 1; preserves server-side label
+ * and property-equality filtering. There is exactly one such split per job.
+ * <li>{@code SHARD}: one split per HugeGraph key-range shard (from {@code
+ * traverser().vertexShards / edgeShards}). Used when parallelism > 1 so shards can be
+ * scanned by multiple readers in parallel. The scan API returns all labels in the range, so
+ * the reader filters by label client-side; server-side property filters are not supported in
+ * this mode (rejected at the factory).
+ * </ul>
+ *
+ * <p>The split also carries resumable progress ({@link #page}, {@link #finished}, {@link
+ * #lastEmittedId}) so a reader can checkpoint mid-scan and continue from the same page after
+ * failover. Identity ({@link #equals}/{@link #hashCode}) is on {@link #splitId} only; the mutable
+ * progress fields are excluded so a split keeps its identity in the enumerator's sets as it
+ * advances.
+ */
+public class HugeGraphSourceSplit implements SourceSplit {
+
+ private static final long serialVersionUID = 1L;
+
+ private final String splitId;
+ private final boolean shardMode;
+ // The label a LABEL_LIST split pages. null for SHARD splits, which scan all labels in a key
+ // range and let the reader filter/route by label client-side.
+ private final String label;
+ // Shard bounds, only meaningful when shardMode == true. Shard itself is not Serializable, so we
+ // store its three primitive components and rebuild it on demand.
+ private final String shardStart;
+ private final String shardEnd;
+ private final long shardLength;
+
+ // Resumable progress. page == null means "from the beginning"; the reader sends it as the empty
+ // string to enter the server's paged mode.
+ private String page;
+ private boolean finished;
+ private String lastEmittedId;
+
+ /** Creates a label-list split that pages exactly {@code label}. */
+ public static HugeGraphSourceSplit labelListSplit(String splitId, String label) {
+ return new HugeGraphSourceSplit(splitId, false, label, null, null, 0L);
+ }
+
+ /** Creates a shard split (parallelism > 1 path). Scans all labels in the key range. */
+ public static HugeGraphSourceSplit shardSplit(String splitId, Shard shard) {
+ return new HugeGraphSourceSplit(
+ splitId, true, null, shard.start(), shard.end(), shard.length());
+ }
+
+ private HugeGraphSourceSplit(
+ String splitId,
+ boolean shardMode,
+ String label,
+ String shardStart,
+ String shardEnd,
+ long shardLength) {
+ this.splitId = splitId;
+ this.shardMode = shardMode;
+ this.label = label;
+ this.shardStart = shardStart;
+ this.shardEnd = shardEnd;
+ this.shardLength = shardLength;
+ this.page = null;
+ this.finished = false;
+ this.lastEmittedId = null;
+ }
+
+ @Override
+ public String splitId() {
+ return splitId;
+ }
+
+ public boolean isShardMode() {
+ return shardMode;
+ }
+
+ /** The label this split pages; null for shard splits (which scan all labels in a range). */
+ public String getLabel() {
+ return label;
+ }
+
+ /** Rebuilds the client {@link Shard} for a shard-mode split. */
+ public Shard toShard() {
+ return new Shard(shardStart, shardEnd, shardLength);
+ }
+
+ public String getPage() {
+ return page;
+ }
+
+ public void setPage(String page) {
+ this.page = page;
+ }
+
+ public boolean isFinished() {
+ return finished;
+ }
+
+ public void setFinished(boolean finished) {
+ this.finished = finished;
+ }
+
+ public String getLastEmittedId() {
+ return lastEmittedId;
+ }
+
+ public void setLastEmittedId(String lastEmittedId) {
+ this.lastEmittedId = lastEmittedId;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ return splitId.equals(((HugeGraphSourceSplit) o).splitId);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(splitId);
+ }
+
+ @Override
+ public String toString() {
+ return "HugeGraphSourceSplit{"
+ + "splitId='"
+ + splitId
+ + '\''
+ + ", shardMode="
+ + shardMode
+ + ", page='"
+ + page
+ + '\''
+ + ", finished="
+ + finished
+ + '}';
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/source/HugeGraphSourceSplitEnumerator.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/source/HugeGraphSourceSplitEnumerator.java
new file mode 100644
index 0000000000..042fbde56b
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/source/HugeGraphSourceSplitEnumerator.java
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.hugegraph.source;
+
+import org.apache.seatunnel.api.source.SourceSplitEnumerator;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.client.HugeGraphClient;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.client.HugeGraphOperations;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.HugeGraphSourceConfig;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.MappingConfig;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorErrorCode;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorException;
+
+import org.apache.hugegraph.structure.graph.Shard;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Supplier;
+
+/**
+ * Assigns read work to HugeGraph source readers.
+ *
+ * <p>Discovery is driven by parallelism:
+ *
+ * <ul>
+ * <li>parallelism == 1: a single {@code LABEL_LIST} split (server-side label/property filtering
+ * preserved).
+ * <li>parallelism > 1: one {@code SHARD} split per key-range shard returned by {@code
+ * traverser().vertexShards / edgeShards}, so readers scan disjoint ranges in parallel.
+ * </ul>
+ *
+ * <p>Splits are assigned exactly once and tracked in {@link #assignedSplits}. On restore the
+ * enumerator does not re-discover; it assigns only the still-unassigned splits and relies on each
+ * reader to resume its already-assigned splits from reader state, so no split is read twice.
+ */
+public class HugeGraphSourceSplitEnumerator
+ implements SourceSplitEnumerator<HugeGraphSourceSplit, HugeGraphSourceState> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HugeGraphSourceSplitEnumerator.class);
+
+ private final Context<HugeGraphSourceSplit> context;
+ private final HugeGraphSourceConfig sourceConfig;
+ private final long splitSize;
+ private final Supplier<HugeGraphOperations> clientFactory;
+ private final Object lock = new Object();
+
+ private final Set<HugeGraphSourceSplit> allSplits = new LinkedHashSet<>();
+ private final Set<HugeGraphSourceSplit> assignedSplits = new HashSet<>();
+ private boolean needsDiscovery;
+
+ public HugeGraphSourceSplitEnumerator(
+ Context<HugeGraphSourceSplit> context,
+ HugeGraphSourceConfig sourceConfig,
+ long splitSize) {
+ this(context, sourceConfig, splitSize, null);
+ }
+
+ public HugeGraphSourceSplitEnumerator(
+ Context<HugeGraphSourceSplit> context,
+ HugeGraphSourceConfig sourceConfig,
+ long splitSize,
+ HugeGraphSourceState restoredState) {
+ this(
+ context,
+ sourceConfig,
+ splitSize,
+ restoredState,
+ () -> new HugeGraphClient(sourceConfig.getConnectionConfig()));
+ }
+
+ HugeGraphSourceSplitEnumerator(
+ Context<HugeGraphSourceSplit> context,
+ HugeGraphSourceConfig sourceConfig,
+ long splitSize,
+ HugeGraphSourceState restoredState,
+ Supplier<HugeGraphOperations> clientFactory) {
+ this.context = context;
+ this.sourceConfig = sourceConfig;
+ this.splitSize = splitSize;
+ this.clientFactory = clientFactory;
+ if (restoredState == null) {
+ this.needsDiscovery = true;
+ } else {
+ this.needsDiscovery = false;
+ this.allSplits.addAll(restoredState.getAllSplits());
+ this.assignedSplits.addAll(restoredState.getAssignedSplits());
+ }
+ }
+
+ @Override
+ public void open() {
+ if (needsDiscovery) {
+ synchronized (lock) {
+ discover();
+ needsDiscovery = false;
+ }
+ }
+ }
+
+ private void discover() {
+ if (sourceConfig.isReadAllLabels()) {
+ for (String label : sourceConfig.getLabels()) {
+ allSplits.add(HugeGraphSourceSplit.labelListSplit("label-list-" + label, label));
+ }
+ LOG.info(
+ "HugeGraph source: read-all-labels, created {} label-list split(s) for {} "
+ + "labels {}",
+ allSplits.size(),
+ sourceConfig.getLabelType(),
+ sourceConfig.getLabels());
+ return;
+ }
+ int parallelism = context.currentParallelism();
+
+ // Runtime guard: the factory-level checkFilterParallelism() reads the per-source
+ // 'parallelism' option, which does not see env { parallelism = N }. This runtime check
+ // catches the combination at the last safe point — before any shard splits are created
+ // — so filter + parallelism > 1 is guaranteed to fail fast.
+ Map<String, Object> filter = sourceConfig.getFilter();
+ boolean hasFilter = filter != null && !filter.isEmpty();
+ if (parallelism > 1 && hasFilter) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "HugeGraph source 'filter' cannot be combined with parallelism > 1 "
+ + "(runtime parallelism is %d): parallel reads use shard "
+ + "key-range scans that do not support server-side property "
+ + "filtering. Either set parallelism to 1 to keep the filter, "
+ + "or remove the filter to read in parallel.",
+ parallelism));
+ }
+
+ if (parallelism <= 1) {
+ allSplits.add(
+ HugeGraphSourceSplit.labelListSplit("label-list", sourceConfig.getLabel()));
+ LOG.info(
+ "HugeGraph source: parallelism=1, using single label-list split for label '{}'",
+ sourceConfig.getLabel());
+ return;
+ }
+ boolean vertex = sourceConfig.getLabelType() == MappingConfig.LabelType.VERTEX;
+ HugeGraphOperations client = clientFactory.get();
+ List<Shard> shards;
+ try {
+ shards = vertex ? client.vertexShards(splitSize) : client.edgeShards(splitSize);
+ } catch (RuntimeException e) {
+ // Shard splitting is a scan-capable-backend feature. The in-memory backend rejects
+ // vertexShards/edgeShards, and the raw server error gives the user no way forward, so
+ // point them at the parallelism=1 label-list path (the original error is kept as
+ // cause).
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.GRAPH_OPERATION_FAILED,
+ String.format(
+ "Failed to split %s label '%s' into shards for a parallel (parallelism>1) "
+ + "read. Shard scans require a scan-capable HugeGraph backend "
+ + "(RocksDB/HBase/Cassandra); the in-memory backend does not "
+ + "support them. Set parallelism=1 to read via a single "
+ + "label-list scan instead.",
+ vertex ? "vertex" : "edge", sourceConfig.getLabel()),
+ e);
+ } finally {
+ client.close();
+ }
+ int index = 0;
+ for (Shard shard : shards) {
+ allSplits.add(HugeGraphSourceSplit.shardSplit("shard-" + index, shard));
+ index++;
+ }
+ LOG.info(
+ "HugeGraph source: parallelism={}, discovered {} shard split(s) for {} label '{}' "
+ + "(split_size={})",
+ parallelism,
+ allSplits.size(),
+ vertex ? "vertex" : "edge",
+ sourceConfig.getLabel(),
+ splitSize);
+ }
+
+ @Override
+ public void run() {
+ synchronized (lock) {
+ int parallelism = context.currentParallelism();
+ List<List<HugeGraphSourceSplit>> perReader = new ArrayList<>();
+ for (int i = 0; i < parallelism; i++) {
+ perReader.add(new ArrayList<>());
+ }
+ int cursor = 0;
+ for (HugeGraphSourceSplit split : allSplits) {
+ if (assignedSplits.contains(split)) {
+ continue;
+ }
+ perReader.get(cursor % parallelism).add(split);
+ cursor++;
+ }
+ for (int subtask = 0; subtask < parallelism; subtask++) {
+ List<HugeGraphSourceSplit> share = perReader.get(subtask);
+ context.assignSplit(subtask, share);
+ assignedSplits.addAll(share);
+ // Bounded source: tell every reader (even those with no splits) that no more
+ // splits are coming, so it can finish once it drains what it was assigned.
+ context.signalNoMoreSplits(subtask);
+ }
+ }
+ }
+
+ @Override
+ public void addSplitsBack(List<HugeGraphSourceSplit> splits, int subtaskId) {
+ if (splits == null || splits.isEmpty()) {
+ return;
+ }
+ synchronized (lock) {
+ assignedSplits.removeAll(splits);
+ context.assignSplit(subtaskId, splits);
+ assignedSplits.addAll(splits);
+ context.signalNoMoreSplits(subtaskId);
+ }
+ }
+
+ @Override
+ public int currentUnassignedSplitSize() {
+ synchronized (lock) {
+ return allSplits.size() - assignedSplits.size();
+ }
+ }
+
+ @Override
+ public void handleSplitRequest(int subtaskId) {
+ // Push model: splits are assigned eagerly in run()/addSplitsBack, not on request.
+ }
+
+ @Override
+ public void registerReader(int subtaskId) {
+ // No-op: assignment happens in run().
+ }
+
+ @Override
+ public HugeGraphSourceState snapshotState(long checkpointId) {
+ synchronized (lock) {
+ return new HugeGraphSourceState(
+ new HashSet<>(allSplits), new HashSet<>(assignedSplits));
+ }
+ }
+
+ @Override
+ public void notifyCheckpointComplete(long checkpointId) {
+ // No-op.
+ }
+
+ @Override
+ public void close() {
+ // The discovery client is opened and closed within discover(); nothing long-lived to close.
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/source/HugeGraphSourceState.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/source/HugeGraphSourceState.java
new file mode 100644
index 0000000000..e336528dcd
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/source/HugeGraphSourceState.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.hugegraph.source;
+
+import java.io.Serializable;
+import java.util.Set;
+
+/**
+ * Enumerator checkpoint state for the HugeGraph source.
+ *
+ * <p>Persists the full discovered split set plus which splits have already been assigned to
+ * readers. On restore the enumerator does not re-query shards (which could return different
+ * boundaries), assigns only the still-unassigned splits, and lets each reader resume its
+ * already-assigned splits from its own reader state.
+ */
+public class HugeGraphSourceState implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final Set<HugeGraphSourceSplit> allSplits;
+ private final Set<HugeGraphSourceSplit> assignedSplits;
+
+ public HugeGraphSourceState(
+ Set<HugeGraphSourceSplit> allSplits, Set<HugeGraphSourceSplit> assignedSplits) {
+ this.allSplits = allSplits;
+ this.assignedSplits = assignedSplits;
+ }
+
+ public Set<HugeGraphSourceSplit> getAllSplits() {
+ return allSplits;
+ }
+
+ public Set<HugeGraphSourceSplit> getAssignedSplits() {
+ return assignedSplits;
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/source/LabelTableContext.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/source/LabelTableContext.java
new file mode 100644
index 0000000000..7593c4e270
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/source/LabelTableContext.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.hugegraph.source;
+
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * Per-label read context for the HugeGraph source. Carries everything the reader needs to turn one
+ * label's elements into routable rows: the property row type (to fill the property columns), the
+ * produced row type (reserved columns + properties), and the {@code tableId} string ({@code
+ * CatalogTable.getTablePath().toString()}) that a downstream MultiTableSink routes on.
+ *
+ * <p>In single-label mode the source builds exactly one context; in read-all mode one per
+ * discovered label. The reader looks the context up by the split's active label.
+ */
+public class LabelTableContext implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final String label;
+ private final SeaTunnelRowType propertyRowType;
+ private final SeaTunnelRowType outputRowType;
+ private final String tableId;
+
+ public LabelTableContext(
+ String label,
+ SeaTunnelRowType propertyRowType,
+ SeaTunnelRowType outputRowType,
+ String tableId) {
+ this.label = Objects.requireNonNull(label);
+ this.propertyRowType = Objects.requireNonNull(propertyRowType);
+ this.outputRowType = Objects.requireNonNull(outputRowType);
+ this.tableId = Objects.requireNonNull(tableId);
+ }
+
+ public String getLabel() {
+ return label;
+ }
+
+ public SeaTunnelRowType getPropertyRowType() {
+ return propertyRowType;
+ }
+
+ public SeaTunnelRowType getOutputRowType() {
+ return outputRowType;
+ }
+
+ public String getTableId() {
+ return tableId;
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/utils/DataTypeUtil.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/utils/DataTypeUtil.java
index 1984a9e2bd..8c70b03b47 100644
--- a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/utils/DataTypeUtil.java
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/utils/DataTypeUtil.java
@@ -17,6 +17,7 @@
package org.apache.seatunnel.connectors.seatunnel.hugegraph.utils;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.ListFormat;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorErrorCode;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorException;
@@ -24,12 +25,14 @@ import org.apache.hugegraph.structure.constant.Cardinality;
import org.apache.hugegraph.structure.constant.DataType;
import org.apache.hugegraph.structure.schema.PropertyKey;
+import java.lang.reflect.Array;
+import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneId;
-import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.LinkedHashSet;
@@ -61,6 +64,32 @@ public final class DataTypeUtil {
public static Object convert(
Object value, PropertyKey propertyKey, String dateFormat, String timeZone) {
+ return convert(
+ value,
+ propertyKey,
+ dateFormat,
+ timeZone,
+ Collections.emptyList(),
+ new ListFormat());
+ }
+
+ public static Object convert(
+ Object value,
+ PropertyKey propertyKey,
+ String dateFormat,
+ String timeZone,
+ ListFormat listFormat) {
+ return convert(
+ value, propertyKey, dateFormat, timeZone, Collections.emptyList(), listFormat);
+ }
+
+ public static Object convert(
+ Object value,
+ PropertyKey propertyKey,
+ String dateFormat,
+ String timeZone,
+ List<String> extraDateFormats,
+ ListFormat listFormat) {
E.checkArgumentNotNull(value, "The value to be converted can't be null");
String key = propertyKey.name();
@@ -68,10 +97,19 @@ public final class DataTypeUtil {
Cardinality cardinality = propertyKey.cardinality();
switch (cardinality) {
case SINGLE:
- return parseSingleValue(key, value, dataType, dateFormat, timeZone);
+ return parseSingleValue(
+ key, value, dataType, dateFormat, timeZone, extraDateFormats);
case SET:
case LIST:
- return parseMultiValues(key, value, dataType, cardinality, dateFormat, timeZone);
+ return parseMultiValues(
+ key,
+ value,
+ dataType,
+ cardinality,
+ dateFormat,
+ timeZone,
+ extraDateFormats,
+ listFormat);
default:
throw new HugeGraphConnectorException(
HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
@@ -89,25 +127,36 @@ public final class DataTypeUtil {
DataType dataType,
Cardinality cardinality,
String dateFormat,
- String timeZone) {
- // JSON file should not parse again
- if (values instanceof Collection
- && checkCollectionDataType(key, (Collection<?>) values, dataType)) {
- return values;
+ String timeZone,
+ List<String> extraDateFormats,
+ ListFormat listFormat) {
+ Collection<?> sourceValues;
+ if (values instanceof Collection) {
+ sourceValues = (Collection<?>) values;
+ } else if (values.getClass().isArray()) {
+ List<Object> arrayValues = new ArrayList<>(Array.getLength(values));
+ for (int i = 0; i < Array.getLength(values); i++) {
+ arrayValues.add(Array.get(values, i));
+ }
+ sourceValues = arrayValues;
+ } else {
+ E.checkState(
+ values instanceof String,
+ "The value(key='%s') must be a Collection, array, or String type, "
+ + "but got '%s'(%s)",
+ key,
+ values,
+ values.getClass());
+ sourceValues = split(key, (String) values, listFormat);
}
- E.checkState(
- values instanceof String,
- "The value(key='%s') must be String type, " + "but got '%s'(%s)",
- key,
- values);
- String rawValue = (String) values;
- List<Object> valueColl = split(key, rawValue);
Collection<Object> results =
cardinality == Cardinality.LIST ? new ArrayList<>() : new LinkedHashSet<>();
- valueColl.forEach(
+ sourceValues.forEach(
value -> {
- results.add(parseSingleValue(key, value, dataType, dateFormat, timeZone));
+ results.add(
+ parseSingleValue(
+ key, value, dataType, dateFormat, timeZone, extraDateFormats));
});
E.checkArgument(
checkCollectionDataType(key, results, dataType),
@@ -124,8 +173,15 @@ public final class DataTypeUtil {
Collection<?> collection = (Collection<?>) rawColumnValue;
return new ArrayList<>(collection);
}
+ if (rawColumnValue.getClass().isArray()) {
+ List<Object> values = new ArrayList<>(Array.getLength(rawColumnValue));
+ for (int i = 0; i < Array.getLength(rawColumnValue); i++) {
+ values.add(Array.get(rawColumnValue, i));
+ }
+ return values;
+ }
String rawValue = rawColumnValue.toString();
- return split(key, rawValue);
+ return split(key, rawValue, new ListFormat());
}
public static UUID parseUUID(String key, Object rawValue) {
@@ -150,7 +206,12 @@ public final class DataTypeUtil {
}
private static Object parseSingleValue(
- String key, Object rawValue, DataType dataType, String dateFormat, String timeZone) {
+ String key,
+ Object rawValue,
+ DataType dataType,
+ String dateFormat,
+ String timeZone,
+ List<String> extraDateFormats) {
Object value = trimString(rawValue);
if (value == null) {
return null;
@@ -166,7 +227,7 @@ public final class DataTypeUtil {
case BOOLEAN:
return parseBoolean(key, value);
case DATE:
- return parseDate(key, value, dateFormat, timeZone);
+ return parseDate(key, value, dateFormat, timeZone, extraDateFormats);
case UUID:
return parseUUID(key, value);
default:
@@ -308,7 +369,12 @@ public final class DataTypeUtil {
key, value, value.getClass()));
}
- private static Date parseDate(String key, Object value, String dateFormat, String timeZone) {
+ private static Date parseDate(
+ String key,
+ Object value,
+ String dateFormat,
+ String timeZone,
+ List<String> extraDateFormats) {
if (value instanceof Date) {
return (Date) value;
}
@@ -351,7 +417,21 @@ public final class DataTypeUtil {
}
}
- if (dateFormat == null || dateFormat.isEmpty()) {
+ // Candidate patterns, primary first then the extras — tried in order (deterministic,
+ // unlike the loader which uses a HashSet), first successful parse wins.
+ List<String> formats = new ArrayList<>();
+ if (dateFormat != null && !dateFormat.isEmpty()) {
+ formats.add(dateFormat);
+ }
+ if (extraDateFormats != null) {
+ for (String extra : extraDateFormats) {
+ if (extra != null && !extra.isEmpty()) {
+ formats.add(extra);
+ }
+ }
+ }
+
+ if (formats.isEmpty()) {
// Fallback for when no format is provided.
try {
return new Date(Long.parseLong(strValue));
@@ -363,19 +443,27 @@ public final class DataTypeUtil {
}
}
- try {
- DateTimeFormatter formatter = DateTimeFormatter.ofPattern(dateFormat);
- LocalDateTime ldt = LocalDateTime.parse(strValue, formatter);
- ZonedDateTime zdt = ldt.atZone(zoneId);
- return Date.from(zdt.toInstant());
- } catch (Exception e) {
- throw new HugeGraphConnectorException(
- HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
- String.format(
- "Failed to parse date string '%s' with format '%s'",
- value, dateFormat),
- e);
+ Exception lastFailure = null;
+ for (String format : formats) {
+ try {
+ DateTimeFormatter formatter = DateTimeFormatter.ofPattern(format);
+ LocalDateTime ldt;
+ try {
+ ldt = LocalDateTime.parse(strValue, formatter);
+ } catch (java.time.format.DateTimeParseException dateTimeFailure) {
+ ldt = LocalDate.parse(strValue, formatter).atStartOfDay();
+ }
+ return Date.from(ldt.atZone(zoneId).toInstant());
+ } catch (Exception e) {
+ lastFailure = e;
+ }
}
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "Failed to parse date string '%s' with any of the formats %s",
+ value, formats),
+ lastFailure);
}
throw new HugeGraphConnectorException(
HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
@@ -384,26 +472,35 @@ public final class DataTypeUtil {
key, value, value.getClass()));
}
- private static List<Object> split(String key, String rawValue) {
+ private static List<Object> split(String key, String rawValue, ListFormat listFormat) {
List<Object> valueColl = new ArrayList<>();
if (rawValue == null || rawValue.isEmpty()) {
return valueColl;
}
String value = rawValue.trim();
- String startSymbol = "[";
- String endSymbol = "]";
- if (value.startsWith(startSymbol) && value.endsWith(endSymbol)) {
+ String startSymbol = listFormat.getStartSymbol();
+ String endSymbol = listFormat.getEndSymbol();
+ if (startSymbol != null
+ && !startSymbol.isEmpty()
+ && endSymbol != null
+ && !endSymbol.isEmpty()
+ && value.startsWith(startSymbol)
+ && value.endsWith(endSymbol)) {
value = value.substring(startSymbol.length(), value.length() - endSymbol.length());
}
- String elemDelimiter = ",";
- // TODO: use a configurable list format
- com.google.common.base.Splitter.on(elemDelimiter)
+ Set<String> ignoredElems = new HashSet<>(listFormat.getIgnoredElems());
+ com.google.common.base.Splitter.on(listFormat.getElemDelimiter())
.trimResults()
.omitEmptyStrings()
.split(value)
- .forEach(valueColl::add);
+ .forEach(
+ elem -> {
+ if (!ignoredElems.contains(elem)) {
+ valueColl.add(elem);
+ }
+ });
return valueColl;
}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/utils/HugeGraphTypeConverter.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/utils/HugeGraphTypeConverter.java
new file mode 100644
index 0000000000..efc9e8ebd8
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/utils/HugeGraphTypeConverter.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.hugegraph.utils;
+
+import org.apache.seatunnel.api.table.type.ArrayType;
+import org.apache.seatunnel.api.table.type.BasicType;
+import org.apache.seatunnel.api.table.type.LocalTimeType;
+import org.apache.seatunnel.api.table.type.PrimitiveByteArrayType;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorErrorCode;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorException;
+
+import org.apache.hugegraph.structure.constant.Cardinality;
+import org.apache.hugegraph.structure.constant.DataType;
+
+/**
+ * Maps HugeGraph property-key types to SeaTunnel types for the source read path. Shared by schema
+ * validation (declared vs. server) and schema auto-discovery (deriving the row type from the server
+ * label when {@code schema} is omitted) so both use identical mapping rules.
+ */
+public final class HugeGraphTypeConverter {
+
+ private HugeGraphTypeConverter() {}
+
+ /**
+ * Maps a HugeGraph property type + cardinality to a SeaTunnel type. A {@code LIST}/{@code SET}
+ * cardinality becomes {@code array<element>}; {@code SINGLE} (or null) stays scalar. The {@code
+ * propertyName} is used only to make any error message point at the offending column.
+ */
+ public static SeaTunnelDataType<?> toSeaTunnelType(
+ DataType dataType, Cardinality cardinality, String propertyName) {
+ SeaTunnelDataType<?> scalar = toSeaTunnelScalarType(dataType, propertyName);
+ if (cardinality == null || cardinality == Cardinality.SINGLE) {
+ return scalar;
+ }
+ // BLOB elements would produce byte[][], which downstream SeaTunnel operators do not
+ // uniformly handle — reject with a clear message rather than a mysterious CCE later.
+ if (dataType == DataType.BLOB) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Property '%s': type BLOB with cardinality %s is not supported for reads.",
+ propertyName, cardinality));
+ }
+ return ArrayType.of(scalar);
+ }
+
+ public static SeaTunnelDataType<?> toSeaTunnelScalarType(
+ DataType dataType, String propertyName) {
+ switch (dataType) {
+ case TEXT:
+ return BasicType.STRING_TYPE;
+ case BYTE:
+ return BasicType.BYTE_TYPE;
+ case INT:
+ return BasicType.INT_TYPE;
+ case LONG:
+ return BasicType.LONG_TYPE;
+ case FLOAT:
+ return BasicType.FLOAT_TYPE;
+ case DOUBLE:
+ return BasicType.DOUBLE_TYPE;
+ case BOOLEAN:
+ return BasicType.BOOLEAN_TYPE;
+ case DATE:
+ return LocalTimeType.LOCAL_DATE_TIME_TYPE;
+ case UUID:
+ return BasicType.STRING_TYPE;
+ case OBJECT:
+ // OBJECT holds an arbitrary serialized value with no fixed shape; read it as its
+ // string representation so a single such column does not block the whole label
+ // read.
+ return BasicType.STRING_TYPE;
+ case BLOB:
+ return PrimitiveByteArrayType.INSTANCE;
+ default:
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Property '%s': unsupported HugeGraph property type for source: %s",
+ propertyName, dataType));
+ }
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/utils/SchemaManager.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/utils/SchemaManager.java
new file mode 100644
index 0000000000..bfed20496b
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/utils/SchemaManager.java
@@ -0,0 +1,555 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.hugegraph.utils;
+
+import org.apache.seatunnel.api.table.type.ArrayType;
+import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
+import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.client.HugeGraphClient;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.HugeGraphSchemaSaveMode;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.LabelOptions;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.MappingConfig;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.MappingConfig.LabelType;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.ReservedColumns;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorErrorCode;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorException;
+
+import org.apache.hugegraph.structure.constant.Cardinality;
+import org.apache.hugegraph.structure.constant.DataType;
+import org.apache.hugegraph.structure.constant.Frequency;
+import org.apache.hugegraph.structure.constant.IdStrategy;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Manages HugeGraph schema lifecycle during Sink initialization. Handles auto-creation under
+ * CREATE_SCHEMA_WHEN_NOT_EXIST and strict validation under ERROR_WHEN_SCHEMA_NOT_EXIST.
+ */
+public final class SchemaManager {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SchemaManager.class);
+
+ private final HugeGraphClient client;
+ private final HugeGraphSchemaSaveMode saveMode;
+ private final SeaTunnelRowType rowType;
+
+ public SchemaManager(
+ HugeGraphClient client, HugeGraphSchemaSaveMode saveMode, SeaTunnelRowType rowType) {
+ this.client = client;
+ this.saveMode = saveMode;
+ this.rowType = rowType;
+ }
+
+ /**
+ * Ensures schema exists for all mappings. Creation order: PropertyKeys first, then
+ * VertexLabels, then EdgeLabels — so edge source/target labels exist before edge creation.
+ */
+ public void ensureSchema(List<MappingConfig> mappings) {
+ if (saveMode == HugeGraphSchemaSaveMode.CREATE_SCHEMA_WHEN_NOT_EXIST) {
+ // Phase 1: create all PropertyKeys
+ for (MappingConfig mapping : mappings) {
+ Set<String> propertyNames = resolveTargetPropertyNames(mapping);
+ createMissingPropertyKeys(mapping, propertyNames);
+ if (mapping.getType() == LabelType.EDGE) {
+ createMissingPropertyKeys(mapping, resolveEndpointPropertyNames(mapping));
+ }
+ }
+ // Phase 2: create all VertexLabels
+ Set<String> resolvedVertexLabels = new HashSet<>();
+ for (MappingConfig mapping : mappings) {
+ if (mapping.getType() == LabelType.VERTEX) {
+ Set<String> propertyNames = resolveTargetPropertyNames(mapping);
+ createVertexLabelIfMissing(mapping, propertyNames);
+ resolvedVertexLabels.add(mapping.getLabel());
+ }
+ }
+ // Edge-only mappings still need reconstructable endpoint vertex schemas.
+ for (MappingConfig mapping : mappings) {
+ if (mapping.getType() == LabelType.EDGE) {
+ createEndpointVertexLabelIfMissing(
+ mapping, mapping.getSourceConfig(), resolvedVertexLabels);
+ createEndpointVertexLabelIfMissing(
+ mapping, mapping.getTargetConfig(), resolvedVertexLabels);
+ }
+ }
+ // Phase 3: create all EdgeLabels (source/target vertex labels now guaranteed to exist)
+ for (MappingConfig mapping : mappings) {
+ if (mapping.getType() == LabelType.EDGE) {
+ Set<String> propertyNames = resolveTargetPropertyNames(mapping);
+ createEdgeLabelIfMissing(mapping, propertyNames);
+ }
+ }
+ } else {
+ for (MappingConfig mapping : mappings) {
+ Set<String> propertyNames = resolveTargetPropertyNames(mapping);
+ validateSchemaExists(mapping, propertyNames);
+ }
+ }
+ }
+
+ private Set<String> resolveEndpointPropertyNames(MappingConfig mapping) {
+ Set<String> propertyNames = new HashSet<>();
+ addEndpointPropertyNames(mapping.getSourceConfig(), mapping, propertyNames);
+ addEndpointPropertyNames(mapping.getTargetConfig(), mapping, propertyNames);
+ return propertyNames;
+ }
+
+ private void addEndpointPropertyNames(
+ MappingConfig.SourceTargetConfig endpoint,
+ MappingConfig mapping,
+ Set<String> propertyNames) {
+ if (endpoint == null || ReservedColumns.isRawIdPassthrough(endpoint.getIdFields())) {
+ // Raw-id passthrough reuses a reserved column (~source_id/~target_id) that is not a
+ // HugeGraph property — never create a PropertyKey for it.
+ return;
+ }
+ List<String> properties =
+ mapToTargetNames(endpoint.getIdFields(), mapping.getFieldMapping());
+ if (properties != null) {
+ propertyNames.addAll(properties);
+ }
+ }
+
+ private void createEndpointVertexLabelIfMissing(
+ MappingConfig edgeMapping,
+ MappingConfig.SourceTargetConfig endpoint,
+ Set<String> resolvedVertexLabels) {
+ if (endpoint == null || resolvedVertexLabels.contains(endpoint.getLabel())) {
+ return;
+ }
+ if (endpoint.getIdFields() == null || endpoint.getIdFields().isEmpty()) {
+ return;
+ }
+ // Raw-id passthrough carries a pre-assembled endpoint id, not primary-key columns, so we
+ // cannot synthesize a PRIMARY_KEY vertex label from it. Such a clone assumes the endpoint
+ // vertex label already exists; leave creation to the user.
+ if (ReservedColumns.isRawIdPassthrough(endpoint.getIdFields())) {
+ return;
+ }
+ if (client.getVertexLabelOrNull(endpoint.getLabel()) != null) {
+ resolvedVertexLabels.add(endpoint.getLabel());
+ return;
+ }
+
+ List<String> primaryKeys =
+ mapToTargetNames(endpoint.getIdFields(), edgeMapping.getFieldMapping());
+ LOG.info(
+ "Mapping[EDGE/{}]: Auto-creating endpoint VertexLabel '{}' with PRIMARY_KEY fields={}",
+ edgeMapping.getLabel(),
+ endpoint.getLabel(),
+ primaryKeys);
+ client.createVertexLabelIfNotExist(
+ endpoint.getLabel(),
+ IdStrategy.PRIMARY_KEY,
+ primaryKeys,
+ new ArrayList<>(primaryKeys),
+ new ArrayList<>(),
+ new LabelOptions(null, null, null, null));
+ resolvedVertexLabels.add(endpoint.getLabel());
+ }
+
+ /** Resolves the set of target property names that will be written for this mapping. */
+ private Set<String> resolveTargetPropertyNames(MappingConfig mapping) {
+ Set<String> result = new HashSet<>();
+ Map<String, String> fieldMapping = mapping.getFieldMapping();
+
+ Set<String> sourceFields = new HashSet<>();
+ if (mapping.getProperties().isEmpty()) {
+ for (String fieldName : rowType.getFieldNames()) {
+ // Reserved columns emitted by HugeGraph Source (~id, ~label, ...) are not valid
+ // HugeGraph property key names — HugeGraph rejects PropertyKey names starting
+ // with '~'. An implicit Source→Sink round-trip would otherwise attempt to create
+ // them and fail at label creation time.
+ if (fieldName != null && !fieldName.startsWith("~")) {
+ sourceFields.add(fieldName);
+ }
+ }
+ if (mapping.getType() == LabelType.EDGE) {
+ removeIdFields(sourceFields, mapping.getSourceConfig());
+ removeIdFields(sourceFields, mapping.getTargetConfig());
+ }
+ // `ignored` blacklist only applies in implicit mode; must match the mappers so schema
+ // creation and writes agree on the property set.
+ sourceFields.removeAll(mapping.getIgnored());
+ } else {
+ sourceFields.addAll(mapping.getProperties());
+ }
+ if (mapping.getType() == LabelType.EDGE) {
+ sourceFields.addAll(mapping.getSortKeys());
+ }
+
+ for (String sourceField : sourceFields) {
+ String targetProp = fieldMapping.getOrDefault(sourceField, sourceField);
+ result.add(targetProp);
+ }
+
+ // PRIMARY_KEY idFields are always written as properties by VertexMapper,
+ // so they must be included regardless of whether properties list is empty
+ if (mapping.getType() == LabelType.VERTEX
+ && mapping.getIdStrategy() == IdStrategy.PRIMARY_KEY
+ && mapping.getIdFields() != null) {
+ for (String idField : mapping.getIdFields()) {
+ String targetProp = fieldMapping.getOrDefault(idField, idField);
+ result.add(targetProp);
+ }
+ }
+
+ return result;
+ }
+
+ private static void removeIdFields(
+ Set<String> fields, MappingConfig.SourceTargetConfig sourceTargetConfig) {
+ if (sourceTargetConfig != null && sourceTargetConfig.getIdFields() != null) {
+ fields.removeAll(sourceTargetConfig.getIdFields());
+ }
+ }
+
+ private void createMissingPropertyKeys(MappingConfig mapping, Set<String> targetPropertyNames) {
+ Map<String, String> fieldMapping = mapping.getFieldMapping();
+
+ for (String targetProp : targetPropertyNames) {
+ if (client.getPropertyKeyOrNull(targetProp) != null) {
+ continue;
+ }
+
+ String sourceField = findSourceField(targetProp, fieldMapping);
+ int fieldIndex = findFieldIndex(sourceField);
+ if (fieldIndex < 0) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Mapping[%s/%s]: Source field '%s' for target property '%s' "
+ + "not found in input row. Available fields: %s",
+ mapping.getType(),
+ mapping.getLabel(),
+ sourceField,
+ targetProp,
+ getFieldNames()));
+ }
+
+ SeaTunnelDataType<?> seaType = rowType.getFieldType(fieldIndex);
+ DataType hgType = inferHugeGraphDataType(seaType, mapping, targetProp);
+ Cardinality cardinality = inferCardinality(seaType);
+
+ LOG.info(
+ "Mapping[{}/{}]: Auto-creating PropertyKey '{}' with type={}, cardinality={}",
+ mapping.getType(),
+ mapping.getLabel(),
+ targetProp,
+ hgType,
+ cardinality);
+ client.createPropertyKeyIfNotExist(targetProp, hgType, cardinality);
+ }
+ }
+
+ private void createVertexLabelIfMissing(MappingConfig mapping, Set<String> propertyNames) {
+ if (client.getVertexLabelOrNull(mapping.getLabel()) != null) {
+ LOG.debug("VertexLabel '{}' already exists, skipping creation.", mapping.getLabel());
+ return;
+ }
+
+ IdStrategy idStrategy =
+ mapping.getIdStrategy() != null ? mapping.getIdStrategy() : IdStrategy.PRIMARY_KEY;
+ // Primary keys must reference target property names (after fieldMapping), matching the
+ // property names used for label creation
+ List<String> primaryKeys =
+ idStrategy == IdStrategy.PRIMARY_KEY
+ ? mapToTargetNames(mapping.getIdFields(), mapping.getFieldMapping())
+ : null;
+
+ List<String> nullableKeys = computeNullableKeys(mapping, propertyNames);
+
+ LOG.info(
+ "Mapping[VERTEX/{}]: Auto-creating VertexLabel with idStrategy={}, properties={}",
+ mapping.getLabel(),
+ idStrategy,
+ propertyNames);
+ client.createVertexLabelIfNotExist(
+ mapping.getLabel(),
+ idStrategy,
+ primaryKeys,
+ new ArrayList<>(propertyNames),
+ nullableKeys,
+ buildLabelOptions(mapping));
+ }
+
+ private void createEdgeLabelIfMissing(MappingConfig mapping, Set<String> propertyNames) {
+ if (client.getEdgeLabelOrNull(mapping.getLabel()) != null) {
+ LOG.debug("EdgeLabel '{}' already exists, skipping creation.", mapping.getLabel());
+ return;
+ }
+
+ E.checkNotNull(mapping.getSourceConfig(), "sourceConfig", "edge mapping");
+ E.checkNotNull(mapping.getTargetConfig(), "targetConfig", "edge mapping");
+
+ Frequency frequency =
+ mapping.getFrequency() != null ? mapping.getFrequency() : Frequency.SINGLE;
+ // Sort keys must reference target property names (after fieldMapping)
+ List<String> sortKeys =
+ frequency == Frequency.MULTIPLE
+ ? mapToTargetNames(mapping.getSortKeys(), mapping.getFieldMapping())
+ : null;
+
+ if (frequency == Frequency.MULTIPLE && (sortKeys == null || sortKeys.isEmpty())) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "Mapping[EDGE/%s]: 'sortKeys' must be specified when frequency is MULTIPLE.",
+ mapping.getLabel()));
+ }
+
+ List<String> nullableKeys = computeNullableKeys(mapping, propertyNames);
+
+ LOG.info(
+ "Mapping[EDGE/{}]: Auto-creating EdgeLabel ({}→{}) with frequency={}, properties={}",
+ mapping.getLabel(),
+ mapping.getSourceConfig().getLabel(),
+ mapping.getTargetConfig().getLabel(),
+ frequency,
+ propertyNames);
+ client.createEdgeLabelIfNotExist(
+ mapping.getLabel(),
+ mapping.getSourceConfig().getLabel(),
+ mapping.getTargetConfig().getLabel(),
+ frequency,
+ sortKeys,
+ new ArrayList<>(propertyNames),
+ nullableKeys,
+ buildLabelOptions(mapping));
+ }
+
+ /**
+ * Collects the optional label attributes (ttl / ttlStartTime / enableLabelIndex / userdata)
+ * from the mapping so they are actually applied at label creation instead of silently ignored.
+ */
+ private LabelOptions buildLabelOptions(MappingConfig mapping) {
+ Boolean enableLabelIndex =
+ mapping.getEnableLabelIndex() == null
+ ? null
+ : Boolean.parseBoolean(mapping.getEnableLabelIndex());
+ return new LabelOptions(
+ mapping.getTtl(),
+ mapping.getTtlStartTime(),
+ enableLabelIndex,
+ mapping.getUserdata());
+ }
+
+ private void validateSchemaExists(MappingConfig mapping, Set<String> targetPropertyNames) {
+ if (mapping.getType() == LabelType.VERTEX) {
+ if (client.getVertexLabelOrNull(mapping.getLabel()) == null) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Mapping[VERTEX/%s]: VertexLabel does not exist in HugeGraph. "
+ + "Create it manually or set schema_save_mode=CREATE_SCHEMA_WHEN_NOT_EXIST.",
+ mapping.getLabel()));
+ }
+ } else {
+ if (client.getEdgeLabelOrNull(mapping.getLabel()) == null) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Mapping[EDGE/%s]: EdgeLabel does not exist in HugeGraph. "
+ + "Create it manually or set schema_save_mode=CREATE_SCHEMA_WHEN_NOT_EXIST.",
+ mapping.getLabel()));
+ }
+ }
+
+ for (String propName : targetPropertyNames) {
+ if (client.getPropertyKeyOrNull(propName) == null) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Mapping[%s/%s]: PropertyKey '%s' does not exist in HugeGraph. "
+ + "Create it manually or set schema_save_mode=CREATE_SCHEMA_WHEN_NOT_EXIST.",
+ mapping.getType(), mapping.getLabel(), propName));
+ }
+ }
+ }
+
+ // --- Type inference ---
+
+ private DataType inferHugeGraphDataType(
+ SeaTunnelDataType<?> seaType, MappingConfig mapping, String propertyName) {
+ switch (seaType.getSqlType()) {
+ case STRING:
+ return DataType.TEXT;
+ case BIGINT:
+ return DataType.LONG;
+ case INT:
+ case TINYINT:
+ case SMALLINT:
+ return DataType.INT;
+ case FLOAT:
+ return DataType.FLOAT;
+ case DOUBLE:
+ return DataType.DOUBLE;
+ case BOOLEAN:
+ return DataType.BOOLEAN;
+ case DATE:
+ case TIMESTAMP:
+ return DataType.DATE;
+ case BYTES:
+ return DataType.BLOB;
+ case ARRAY:
+ SeaTunnelDataType<?> elementType = ((ArrayType<?, ?>) seaType).getElementType();
+ return inferHugeGraphDataType(elementType, mapping, propertyName);
+ case MAP:
+ case ROW:
+ case DECIMAL:
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Mapping[%s/%s]: Source field for target property '%s' has unsupported "
+ + "SeaTunnel type '%s' and cannot be auto-created in HugeGraph. "
+ + "Pre-create the PropertyKey with an appropriate representation "
+ + "(for example TEXT for serialized data), or use a Transform to "
+ + "convert the field before the HugeGraph sink.",
+ mapping.getType(),
+ mapping.getLabel(),
+ propertyName,
+ seaType.getSqlType()));
+ default:
+ return DataType.TEXT;
+ }
+ }
+
+ private Cardinality inferCardinality(SeaTunnelDataType<?> seaType) {
+ if (seaType.getSqlType() == org.apache.seatunnel.api.table.type.SqlType.ARRAY) {
+ return Cardinality.LIST;
+ }
+ return Cardinality.SINGLE;
+ }
+
+ // --- Helpers ---
+
+ /** Maps source field names to target property names via fieldMapping. */
+ private static List<String> mapToTargetNames(
+ List<String> sourceFields, Map<String, String> fieldMapping) {
+ if (sourceFields == null) {
+ return null;
+ }
+ List<String> result = new ArrayList<>(sourceFields.size());
+ for (String field : sourceFields) {
+ result.add(fieldMapping.getOrDefault(field, field));
+ }
+ return result;
+ }
+
+ private String findSourceField(String targetProp, Map<String, String> fieldMapping) {
+ for (Map.Entry<String, String> entry : fieldMapping.entrySet()) {
+ if (targetProp.equals(entry.getValue())) {
+ return entry.getKey();
+ }
+ }
+ return targetProp;
+ }
+
+ private int findFieldIndex(String fieldName) {
+ for (int i = 0; i < rowType.getTotalFields(); i++) {
+ if (rowType.getFieldName(i).equals(fieldName)) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ private String getFieldNames() {
+ StringBuilder sb = new StringBuilder("[");
+ for (int i = 0; i < rowType.getTotalFields(); i++) {
+ if (i > 0) {
+ sb.append(", ");
+ }
+ sb.append(rowType.getFieldName(i));
+ }
+ return sb.append("]").toString();
+ }
+
+ /**
+ * Decides which target properties are declared nullable on a newly created label.
+ *
+ * <p>Default (when the mapping declares neither {@code nullableKeys} nor {@code
+ * notNullableKeys}): every non-key property is nullable. HugeGraph server rejects any insert
+ * whose row omits a non-nullable property, so the previous "everything non-null" default meant
+ * a single null cell from JDBC/CSV/Kafka failed an entire batch. Loader and spark-connector
+ * both default to nullable non-key properties; this matches them.
+ *
+ * <p>Key properties — primary keys (PRIMARY_KEY vertices) and MULTIPLE-edge sort keys — are
+ * always excluded because HugeGraph server disallows them being nullable.
+ *
+ * <p>Explicit {@code nullableKeys} wins verbatim (subject to key-exclusion + presence
+ * filtering). If it is empty, {@code notNullableKeys} carves out required-property opt-outs
+ * from the default.
+ */
+ static List<String> computeNullableKeys(MappingConfig mapping, Set<String> propertyNames) {
+ Map<String, String> fm = mapping.getFieldMapping();
+ Set<String> keyProps = computeKeyProperties(mapping);
+
+ List<String> explicit = mapping.getNullableKeys();
+ if (!explicit.isEmpty()) {
+ List<String> result = new ArrayList<>();
+ for (String nk : explicit) {
+ String targetName = fm.getOrDefault(nk, nk);
+ if (propertyNames.contains(targetName) && !keyProps.contains(targetName)) {
+ result.add(targetName);
+ }
+ }
+ return result;
+ }
+
+ Set<String> notNullableTargets = new HashSet<>();
+ for (String nk : mapping.getNotNullableKeys()) {
+ notNullableTargets.add(fm.getOrDefault(nk, nk));
+ }
+
+ List<String> result = new ArrayList<>();
+ for (String propName : propertyNames) {
+ if (!keyProps.contains(propName) && !notNullableTargets.contains(propName)) {
+ result.add(propName);
+ }
+ }
+ return result;
+ }
+
+ private static Set<String> computeKeyProperties(MappingConfig mapping) {
+ Set<String> keys = new HashSet<>();
+ Map<String, String> fm = mapping.getFieldMapping();
+ if (mapping.getType() == LabelType.VERTEX
+ && mapping.getIdStrategy() == IdStrategy.PRIMARY_KEY
+ && mapping.getIdFields() != null) {
+ for (String field : mapping.getIdFields()) {
+ keys.add(fm.getOrDefault(field, field));
+ }
+ }
+ if (mapping.getType() == LabelType.EDGE
+ && mapping.getFrequency() == Frequency.MULTIPLE
+ && mapping.getSortKeys() != null) {
+ for (String field : mapping.getSortKeys()) {
+ keys.add(fm.getOrDefault(field, field));
+ }
+ }
+ return keys;
+ }
+}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/utils/SchemaValidator.java b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/utils/SchemaValidator.java
index 52cd49ddfd..630e9d0a1b 100644
--- a/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/utils/SchemaValidator.java
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/main/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/utils/SchemaValidator.java
@@ -21,142 +21,475 @@ import org.apache.seatunnel.api.table.type.ArrayType;
import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.client.HugeGraphClient;
-import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.HugeGraphSinkConfig;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.MappingConfig;
-import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.SchemaConfig;
-import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.SchemaConfig.LabelType;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.MappingConfig.LabelType;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.ReservedColumns;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorErrorCode;
import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorException;
import org.apache.hugegraph.structure.constant.Cardinality;
import org.apache.hugegraph.structure.constant.DataType;
+import org.apache.hugegraph.structure.constant.Frequency;
+import org.apache.hugegraph.structure.constant.IdStrategy;
import org.apache.hugegraph.structure.schema.EdgeLabel;
import org.apache.hugegraph.structure.schema.PropertyKey;
import org.apache.hugegraph.structure.schema.VertexLabel;
-import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
-/** Validates the SeaTunnel schema against the HugeGraph schema. */
+/**
+ * Validates the connector configuration against the HugeGraph server schema. Validation is
+ * per-mapping: only fields involved in each mapping are checked.
+ */
public final class SchemaValidator {
- private final HugeGraphSinkConfig sinkConfig;
- private final SeaTunnelRowType rowType;
private final HugeGraphClient client;
+ private final SeaTunnelRowType rowType;
- public SchemaValidator(HugeGraphSinkConfig config, SeaTunnelRowType rowType) {
- this.sinkConfig = config;
+ public SchemaValidator(HugeGraphClient client, SeaTunnelRowType rowType) {
+ this.client = client;
this.rowType = rowType;
- this.client = new HugeGraphClient(sinkConfig);
}
- public void validateSchema() {
- try {
- SchemaConfig schemaConfig = sinkConfig.getSchemaConfig();
- if (schemaConfig.getType() == LabelType.VERTEX) {
- validateVertex(schemaConfig);
- } else if (schemaConfig.getType() == LabelType.EDGE) {
- validateEdge(schemaConfig);
+ public void validate(List<MappingConfig> mappings) {
+ for (MappingConfig mapping : mappings) {
+ validateMapping(mapping);
+ }
+ }
+
+ /**
+ * Runs only the config-level checks that do not touch the server, so a job with a malformed
+ * mapping fails before any schema is persisted to HugeGraph. HugeGraph label DDL is
+ * non-transactional and its primary keys / sort keys / frequency are effectively immutable, so
+ * persisting a property key or vertex label and then failing config validation would leave a
+ * schema fragment the user cannot fix in place.
+ */
+ public void validateConfigOnly(List<MappingConfig> mappings) {
+ for (MappingConfig mapping : mappings) {
+ validateMappingConfig(mapping);
+ }
+ }
+
+ /**
+ * Fails fast — BEFORE any schema is created — when a mapping targets a label that already
+ * exists on the server with incompatible immutable attributes (vertex id strategy / primary
+ * keys, edge frequency / sort keys / endpoints). HugeGraph cannot ALTER these attributes, and
+ * {@code ensureSchema} creates the PropertyKeys and labels for the <em>other</em> mappings
+ * first — so catching such a mismatch only in the post-create {@link #validate} would leave
+ * those creations behind as schema pollution and trap the user in a retry loop that never
+ * reconciles. Running this read-only check up front means an incompatible pre-existing label
+ * aborts the job with zero new writes; the fix is to drop that label on the server and re-run.
+ *
+ * <p>Only labels that already exist are inspected; missing ones are left for {@code
+ * ensureSchema} to create. Edge endpoint identity is intentionally not checked here (the
+ * endpoint vertex labels may not exist yet under CREATE_SCHEMA_WHEN_NOT_EXIST); it is verified
+ * afterwards by {@link #validate}.
+ */
+ public void validateExistingLabels(List<MappingConfig> mappings) {
+ for (MappingConfig mapping : mappings) {
+ if (mapping.getType() == LabelType.VERTEX) {
+ if (client.getVertexLabelOrNull(mapping.getLabel()) != null) {
+ validateVertexMapping(mapping);
+ }
} else {
- throw new HugeGraphConnectorException(
- HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
- "Unsupported schema type: " + schemaConfig.getType());
+ EdgeLabel existing = client.getEdgeLabelOrNull(mapping.getLabel());
+ if (existing != null) {
+ validateExistingEdgeLabel(mapping, existing);
+ }
}
- } catch (Exception e) {
- throw e;
- } finally {
- client.close();
}
}
- private void validateVertex(SchemaConfig schemaConfig) {
- String label = schemaConfig.getLabel();
- VertexLabel vertexLabel = this.client.getVertexLabel(label);
- if (vertexLabel == null) {
+ /**
+ * Checks an already-existing EdgeLabel's immutable attributes (frequency, sort keys, source /
+ * target labels) against the config. Deliberately omits the endpoint-vertex identity checks
+ * that {@link #validateEdgeMapping} performs, because those require the endpoint labels to
+ * already exist — which is not guaranteed before {@code ensureSchema} runs.
+ */
+ private void validateExistingEdgeLabel(MappingConfig mapping, EdgeLabel edgeLabel) {
+ String label = mapping.getLabel();
+ Frequency configuredFrequency =
+ mapping.getFrequency() == null ? Frequency.SINGLE : mapping.getFrequency();
+ if (edgeLabel.frequency() != configuredFrequency) {
throw new HugeGraphConnectorException(
HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
- String.format("Vertex label '%s' does not exist in HugeGraph.", label));
+ String.format(
+ "Mapping[EDGE/%s]: frequency mismatch — server='%s', config='%s'. The "
+ + "EdgeLabel already exists with an immutable frequency; drop it "
+ + "on the server before re-running.",
+ label, edgeLabel.frequency(), configuredFrequency));
+ }
+ List<String> configuredSortKeys =
+ configuredFrequency == Frequency.MULTIPLE
+ ? mapToTargetNames(mapping.getSortKeys(), mapping.getFieldMapping())
+ : java.util.Collections.emptyList();
+ if (!edgeLabel.sortKeys().equals(configuredSortKeys)) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Mapping[EDGE/%s]: sort key mismatch — server='%s', config='%s'. The "
+ + "EdgeLabel already exists with immutable sort keys; drop it on "
+ + "the server before re-running.",
+ label, edgeLabel.sortKeys(), configuredSortKeys));
+ }
+ if (!edgeLabel.sourceLabel().equals(mapping.getSourceConfig().getLabel())) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Mapping[EDGE/%s]: sourceLabel mismatch — server='%s', config='%s'. The "
+ + "EdgeLabel already exists; drop it on the server before re-running.",
+ label, edgeLabel.sourceLabel(), mapping.getSourceConfig().getLabel()));
+ }
+ if (!edgeLabel.targetLabel().equals(mapping.getTargetConfig().getLabel())) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Mapping[EDGE/%s]: targetLabel mismatch — server='%s', config='%s'. The "
+ + "EdgeLabel already exists; drop it on the server before re-running.",
+ label, edgeLabel.targetLabel(), mapping.getTargetConfig().getLabel()));
+ }
+ }
+
+ private void validateMapping(MappingConfig mapping) {
+ validateMappingConfig(mapping);
+ if (mapping.getType() == LabelType.VERTEX) {
+ validateVertexMapping(mapping);
+ } else {
+ validateEdgeMapping(mapping);
}
- validateLabelProperties(label, schemaConfig, vertexLabel.properties());
}
- private void validateEdge(SchemaConfig schemaConfig) {
- String label = schemaConfig.getLabel();
- EdgeLabel edgeLabel = this.client.getEdgeLabel(label);
- if (edgeLabel == null) {
+ private void validateMappingConfig(MappingConfig mapping) {
+ E.checkNotNull(mapping.getType(), "type", "mapping");
+ E.checkNotNull(mapping.getLabel(), "label", "mapping");
+
+ // nullableKeys and notNullableKeys are two opposite ways to steer nullability of an
+ // auto-created label. Setting both is ambiguous — notNullableKeys is silently ignored once
+ // an explicit nullableKeys allow-list is present — so reject it up front instead.
+ if (!mapping.getNullableKeys().isEmpty() && !mapping.getNotNullableKeys().isEmpty()) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "Mapping[%s/%s]: 'nullableKeys' and 'notNullableKeys' are mutually "
+ + "exclusive — set at most one.",
+ mapping.getType(), mapping.getLabel()));
+ }
+
+ // `properties` (selected whitelist) and `ignored` (blacklist) are opposite ways to choose
+ // the property set; setting both is ambiguous.
+ if (!mapping.getProperties().isEmpty() && !mapping.getIgnored().isEmpty()) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "Mapping[%s/%s]: 'properties' (selected) and 'ignored' are mutually "
+ + "exclusive — set at most one.",
+ mapping.getType(), mapping.getLabel()));
+ }
+ validateSourceFields(mapping, mapping.getIgnored(), "ignored");
+
+ if (mapping.getType() == LabelType.VERTEX) {
+ E.checkNotNull(
+ mapping.getIdStrategy(),
+ "idStrategy",
+ String.format("mapping[VERTEX/%s]", mapping.getLabel()));
+ if (mapping.getIdStrategy() != IdStrategy.AUTOMATIC) {
+ E.checkNotEmpty(
+ mapping.getIdFields(),
+ "idFields",
+ String.format("mapping[VERTEX/%s]", mapping.getLabel()));
+ validateSourceFields(mapping, mapping.getIdFields(), "idFields");
+ // A vertex that reuses the reserved ~id column supplies the id externally, which
+ // only CUSTOMIZE_* strategies accept. PRIMARY_KEY derives its id from property
+ // values (use those columns instead) and AUTOMATIC is server-assigned.
+ if (ReservedColumns.isRawIdPassthrough(mapping.getIdFields())
+ && mapping.getIdStrategy() != IdStrategy.CUSTOMIZE_STRING
+ && mapping.getIdStrategy() != IdStrategy.CUSTOMIZE_NUMBER
+ && mapping.getIdStrategy() != IdStrategy.CUSTOMIZE_UUID) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "Mapping[VERTEX/%s]: idFields '%s' (raw-id passthrough) requires a "
+ + "CUSTOMIZE_STRING/NUMBER/UUID id strategy, but got '%s'.",
+ mapping.getLabel(),
+ mapping.getIdFields().get(0),
+ mapping.getIdStrategy()));
+ }
+ }
+ if (mapping.isUnfold()) {
+ validateUnfoldable(
+ mapping,
+ mapping.getIdStrategy(),
+ mapping.getIdFields(),
+ String.format("mapping[VERTEX/%s]", mapping.getLabel()));
+ }
+ } else {
+ E.checkNotNull(
+ mapping.getSourceConfig(),
+ "sourceConfig",
+ String.format("mapping[EDGE/%s]", mapping.getLabel()));
+ E.checkNotNull(
+ mapping.getTargetConfig(),
+ "targetConfig",
+ String.format("mapping[EDGE/%s]", mapping.getLabel()));
+ E.checkNotNull(
+ mapping.getSourceConfig().getLabel(),
+ "sourceConfig.label",
+ String.format("mapping[EDGE/%s]", mapping.getLabel()));
+ E.checkNotNull(
+ mapping.getTargetConfig().getLabel(),
+ "targetConfig.label",
+ String.format("mapping[EDGE/%s]", mapping.getLabel()));
+ E.checkNotEmpty(
+ mapping.getSourceConfig().getIdFields(),
+ "sourceConfig.idFields",
+ String.format("mapping[EDGE/%s]", mapping.getLabel()));
+ E.checkNotEmpty(
+ mapping.getTargetConfig().getIdFields(),
+ "targetConfig.idFields",
+ String.format("mapping[EDGE/%s]", mapping.getLabel()));
+ validateSourceFields(
+ mapping, mapping.getSourceConfig().getIdFields(), "sourceConfig.idFields");
+ validateSourceFields(
+ mapping, mapping.getTargetConfig().getIdFields(), "targetConfig.idFields");
+
+ if (mapping.getFrequency() == Frequency.MULTIPLE) {
+ E.checkNotEmpty(
+ mapping.getSortKeys(),
+ "sortKeys",
+ String.format(
+ "mapping[EDGE/%s] with frequency=MULTIPLE", mapping.getLabel()));
+ validateSourceFields(mapping, mapping.getSortKeys(), "sortKeys");
+ }
+ // Endpoint id strategy lives on the server vertex label (unknown at config time), so
+ // here we only enforce the config-derivable rules; the CUSTOMIZE-endpoint requirement
+ // is
+ // enforced at runtime when building ids.
+ if (mapping.isUnfoldSource()) {
+ validateUnfoldable(
+ mapping,
+ null,
+ mapping.getSourceConfig().getIdFields(),
+ String.format("mapping[EDGE/%s] sourceConfig", mapping.getLabel()));
+ }
+ if (mapping.isUnfoldTarget()) {
+ validateUnfoldable(
+ mapping,
+ null,
+ mapping.getTargetConfig().getIdFields(),
+ String.format("mapping[EDGE/%s] targetConfig", mapping.getLabel()));
+ }
+ }
+ validateSourceFields(mapping, mapping.getProperties(), "properties");
+ }
+
+ /**
+ * unfold expands a single list-valued id cell into multiple elements, so it requires exactly
+ * one id field and cannot be combined with raw-id passthrough. When {@code strategy} is known
+ * (vertex), it must be a CUSTOMIZE_* strategy; for edge endpoints the strategy is server-side
+ * and checked when ids are built.
+ */
+ private static void validateUnfoldable(
+ MappingConfig mapping, IdStrategy strategy, List<String> idFields, String context) {
+ if (idFields == null || idFields.size() != 1) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "%s: unfold requires exactly one id field, but got %s.",
+ context, idFields));
+ }
+ if (ReservedColumns.isRawIdPassthrough(idFields)) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "%s: unfold cannot be combined with raw-id passthrough (%s).",
+ context, idFields.get(0)));
+ }
+ if (strategy != null
+ && strategy != IdStrategy.CUSTOMIZE_STRING
+ && strategy != IdStrategy.CUSTOMIZE_NUMBER
+ && strategy != IdStrategy.CUSTOMIZE_UUID) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "%s: unfold requires a CUSTOMIZE_STRING/NUMBER/UUID id strategy, but got '%s'.",
+ context, strategy));
+ }
+ }
+
+ private void validateVertexMapping(MappingConfig mapping) {
+ String label = mapping.getLabel();
+ VertexLabel vertexLabel = client.getVertexLabel(label);
+ if (vertexLabel.idStrategy() != mapping.getIdStrategy()) {
throw new HugeGraphConnectorException(
HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
- String.format("Edge label '%s' does not exist in HugeGraph.", label));
+ String.format(
+ "Mapping[VERTEX/%s]: idStrategy mismatch — server='%s', config='%s'",
+ label, vertexLabel.idStrategy(), mapping.getIdStrategy()));
+ }
+ if (mapping.getIdStrategy() == IdStrategy.PRIMARY_KEY) {
+ List<String> configuredPrimaryKeys =
+ mapToTargetNames(mapping.getIdFields(), mapping.getFieldMapping());
+ if (!vertexLabel.primaryKeys().equals(configuredPrimaryKeys)) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Mapping[VERTEX/%s]: primary key mismatch — server='%s', config='%s'",
+ label, vertexLabel.primaryKeys(), configuredPrimaryKeys));
+ }
+ }
+
+ Set<String> hgProperties = vertexLabel.properties();
+ Set<String> targetProperties = resolveTargetProperties(mapping);
+
+ // PRIMARY_KEY idFields are always included
+ if (mapping.getIdStrategy() == IdStrategy.PRIMARY_KEY && mapping.getIdFields() != null) {
+ Map<String, String> fm = mapping.getFieldMapping();
+ for (String idField : mapping.getIdFields()) {
+ targetProperties.add(fm.getOrDefault(idField, idField));
+ }
+ }
+
+ for (String propName : targetProperties) {
+ validateProperty(label, propName, hgProperties, mapping);
}
- validateSourceTarget(schemaConfig, edgeLabel);
- validateLabelProperties(label, schemaConfig, edgeLabel.properties());
}
- private void validateSourceTarget(SchemaConfig schemaConfig, EdgeLabel edgeLabel) {
- String label = schemaConfig.getLabel();
- String schemaSource = edgeLabel.sourceLabel();
- if (!schemaSource.equals(schemaConfig.getSourceConfig().getLabel())) {
+ private void validateEdgeMapping(MappingConfig mapping) {
+ String label = mapping.getLabel();
+ EdgeLabel edgeLabel = client.getEdgeLabel(label);
+ Frequency configuredFrequency =
+ mapping.getFrequency() == null ? Frequency.SINGLE : mapping.getFrequency();
+ if (edgeLabel.frequency() != configuredFrequency) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Mapping[EDGE/%s]: frequency mismatch — server='%s', config='%s'",
+ label, edgeLabel.frequency(), configuredFrequency));
+ }
+ List<String> configuredSortKeys =
+ configuredFrequency == Frequency.MULTIPLE
+ ? mapToTargetNames(mapping.getSortKeys(), mapping.getFieldMapping())
+ : java.util.Collections.emptyList();
+ if (!edgeLabel.sortKeys().equals(configuredSortKeys)) {
throw new HugeGraphConnectorException(
HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
String.format(
- "EdgeLabel[%s] sourceLabel mismatch: schema=%s, config=%s",
- label, schemaSource, schemaConfig.getSourceConfig()));
+ "Mapping[EDGE/%s]: sort key mismatch — server='%s', config='%s'",
+ label, edgeLabel.sortKeys(), configuredSortKeys));
}
- String schemaTarget = edgeLabel.targetLabel();
- if (!schemaTarget.equals(schemaConfig.getTargetConfig().getLabel())) {
+ // Validate source/target labels match
+ if (!edgeLabel.sourceLabel().equals(mapping.getSourceConfig().getLabel())) {
throw new HugeGraphConnectorException(
HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
String.format(
- "EdgeLabel[%s] sourceLabel mismatch: schema=%s, config=%s",
- label, schemaSource, schemaConfig.getSourceConfig()));
+ "Mapping[EDGE/%s]: sourceLabel mismatch — server='%s', config='%s'",
+ label, edgeLabel.sourceLabel(), mapping.getSourceConfig().getLabel()));
+ }
+ if (!edgeLabel.targetLabel().equals(mapping.getTargetConfig().getLabel())) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Mapping[EDGE/%s]: targetLabel mismatch — server='%s', config='%s'",
+ label, edgeLabel.targetLabel(), mapping.getTargetConfig().getLabel()));
+ }
+ validateEndpointIdentity(mapping, mapping.getSourceConfig(), "sourceConfig");
+ validateEndpointIdentity(mapping, mapping.getTargetConfig(), "targetConfig");
+
+ Set<String> hgProperties = edgeLabel.properties();
+ Set<String> targetProperties = resolveTargetProperties(mapping);
+
+ // Edge source/target idFields are NOT edge properties (unless explicitly in properties)
+ for (String propName : targetProperties) {
+ validateProperty(label, propName, hgProperties, mapping);
}
}
/**
- * Validates if the properties from SeaTunnelRowType are compatible with the HugeGraph schema.
+ * Resolves the set of target property names for validation. Only includes fields listed in
+ * mapping.properties (after fieldMapping transformation). Reserved columns (~id, ~label, ...)
+ * emitted by the HugeGraph Source are excluded — they are not HugeGraph property keys and must
+ * not be validated as such.
*/
- private void validateLabelProperties(
- String label, SchemaConfig schemaConfig, Set<String> hugegraphProperties) {
+ private Set<String> resolveTargetProperties(MappingConfig mapping) {
+ Set<String> result = new HashSet<>();
+ Map<String, String> fieldMapping = mapping.getFieldMapping();
- MappingConfig mappingConfig = schemaConfig.getMapping();
- Map<String, String> fieldMapping =
- mappingConfig == null || mappingConfig.getFieldMapping() == null
- ? Collections.emptyMap()
- : mappingConfig.getFieldMapping();
+ Set<String> sourceFields = new HashSet<>();
+ if (mapping.getProperties().isEmpty()) {
+ for (String fieldName : rowType.getFieldNames()) {
+ sourceFields.add(fieldName);
+ }
+ if (mapping.getType() == LabelType.EDGE) {
+ removeIdFields(sourceFields, mapping.getSourceConfig());
+ removeIdFields(sourceFields, mapping.getTargetConfig());
+ }
+ sourceFields.removeAll(mapping.getIgnored());
+ // Reserved columns (~id, ~label, ~source_id, ~target_id, ~source_label, ~target_label)
+ // are emitted by the HugeGraph Source as routing/passthrough columns, not as
+ // HugeGraph property keys. VertexMapper.applyProperties skips them; the validator
+ // must do the same or it will fail trying to getPropertyKey("~id") from the server.
+ ReservedColumns.stripReserved(sourceFields);
+ } else {
+ sourceFields.addAll(mapping.getProperties());
+ }
+ if (mapping.getType() == LabelType.EDGE) {
+ sourceFields.addAll(mapping.getSortKeys());
+ }
- for (int i = 0; i < rowType.getTotalFields(); i++) {
- String fieldName = rowType.getFieldName(i);
- SeaTunnelDataType<?> seaTunnelType = rowType.getFieldType(i);
- String propertyName = fieldMapping.getOrDefault(fieldName, fieldName);
+ for (String sourceField : sourceFields) {
+ result.add(fieldMapping.getOrDefault(sourceField, sourceField));
+ }
+ return result;
+ }
- // 1. Check if the property exists in HugeGraph
- if (!hugegraphProperties.contains(propertyName)) {
- throw new HugeGraphConnectorException(
- HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
- String.format(
- "Property '%s' for label '%s' is defined in the connector config, but does not exist in the HugeGraph schema.",
- propertyName, label));
- }
+ private static void removeIdFields(
+ Set<String> fields, MappingConfig.SourceTargetConfig sourceTargetConfig) {
+ if (sourceTargetConfig != null && sourceTargetConfig.getIdFields() != null) {
+ fields.removeAll(sourceTargetConfig.getIdFields());
+ }
+ }
- // 2. Check for data type compatibility
- PropertyKey propertyKey = this.client.getPropertyKey(propertyName);
- DataType hugeGraphType = propertyKey.dataType();
- Cardinality cardinality = propertyKey.cardinality();
+ private void validateProperty(
+ String label, String propName, Set<String> hgProperties, MappingConfig mapping) {
+ if (!hgProperties.contains(propName)) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Mapping[%s/%s]: Property '%s' does not exist in HugeGraph schema. "
+ + "Available properties for label '%s': %s",
+ mapping.getType(), label, propName, label, hgProperties));
+ }
- if (!isCompatible(seaTunnelType, hugeGraphType, cardinality)) {
- throw new HugeGraphConnectorException(
- HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
- String.format(
- "Data type mismatch for property '%s' on label '%s'. "
- + "SeaTunnel type '%s' is not compatible with HugeGraph type '%s'.",
- propertyName, label, seaTunnelType, hugeGraphType));
- }
+ // Find source field to check type compatibility
+ String sourceField = findSourceField(propName, mapping.getFieldMapping());
+ int fieldIndex = findFieldIndex(sourceField);
+ if (fieldIndex < 0) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Mapping[%s/%s]: Source field '%s' for property '%s' does not exist in input row",
+ mapping.getType(), label, sourceField, propName));
+ }
+
+ SeaTunnelDataType<?> seaType = rowType.getFieldType(fieldIndex);
+ PropertyKey propertyKey = client.getPropertyKey(propName);
+ DataType hgType = propertyKey.dataType();
+ Cardinality cardinality = propertyKey.cardinality();
+
+ if (!isCompatible(seaType, hgType, cardinality)) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Mapping[%s/%s]: Type mismatch for property '%s' — "
+ + "SeaTunnel type '%s' is not compatible with HugeGraph type '%s' (cardinality=%s).",
+ mapping.getType(), label, propName, seaType, hgType, cardinality));
}
}
- /** Checks if a SeaTunnelDataType is compatible with a HugeGraph DataType. */
private boolean isCompatible(
SeaTunnelDataType<?> seaTunnelType, DataType hugeGraphType, Cardinality cardinality) {
switch (seaTunnelType.getSqlType()) {
@@ -193,8 +526,91 @@ public final class SchemaValidator {
case STRING:
return hugeGraphType == DataType.TEXT;
default:
- // Unsupported types are considered incompatible.
return false;
}
}
+
+ private String findSourceField(String targetProp, Map<String, String> fieldMapping) {
+ for (Map.Entry<String, String> entry : fieldMapping.entrySet()) {
+ if (targetProp.equals(entry.getValue())) {
+ return entry.getKey();
+ }
+ }
+ return targetProp;
+ }
+
+ private void validateEndpointIdentity(
+ MappingConfig mapping, MappingConfig.SourceTargetConfig endpoint, String endpointName) {
+ VertexLabel vertexLabel = client.getVertexLabel(endpoint.getLabel());
+ // Raw-id passthrough reuses the pre-assembled ~source_id/~target_id string and never
+ // rebuilds the endpoint vertex, so there is nothing to match against the label's primary
+ // keys. Requiring the label to exist (getVertexLabel above) is enough.
+ if (ReservedColumns.isRawIdPassthrough(endpoint.getIdFields())) {
+ return;
+ }
+ IdStrategy idStrategy = vertexLabel.idStrategy();
+ List<String> idFields = endpoint.getIdFields();
+ if (idStrategy == IdStrategy.AUTOMATIC) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Mapping[EDGE/%s]: %s label '%s' uses AUTOMATIC IDs, which cannot be reconstructed from input fields",
+ mapping.getLabel(), endpointName, endpoint.getLabel()));
+ }
+ if (idStrategy == IdStrategy.PRIMARY_KEY) {
+ List<String> configuredPrimaryKeys =
+ mapToTargetNames(idFields, mapping.getFieldMapping());
+ if (!vertexLabel.primaryKeys().equals(configuredPrimaryKeys)) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.INVALID_GRAPH_SCHEMA,
+ String.format(
+ "Mapping[EDGE/%s]: %s primary key mismatch for label '%s' — server='%s', config='%s'",
+ mapping.getLabel(),
+ endpointName,
+ endpoint.getLabel(),
+ vertexLabel.primaryKeys(),
+ configuredPrimaryKeys));
+ }
+ } else if ((idStrategy == IdStrategy.CUSTOMIZE_NUMBER
+ || idStrategy == IdStrategy.CUSTOMIZE_UUID)
+ && idFields.size() != 1) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "Mapping[EDGE/%s]: %s for %s requires exactly one id field, but got %s",
+ mapping.getLabel(), endpointName, idStrategy, idFields.size()));
+ }
+ }
+
+ private static List<String> mapToTargetNames(
+ List<String> sourceFields, Map<String, String> fieldMapping) {
+ return sourceFields.stream()
+ .map(field -> fieldMapping.getOrDefault(field, field))
+ .collect(java.util.stream.Collectors.toList());
+ }
+
+ private int findFieldIndex(String fieldName) {
+ for (int i = 0; i < rowType.getTotalFields(); i++) {
+ if (rowType.getFieldName(i).equals(fieldName)) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ private void validateSourceFields(
+ MappingConfig mapping, List<String> sourceFields, String optionName) {
+ if (sourceFields == null) {
+ return;
+ }
+ for (String sourceField : sourceFields) {
+ if (findFieldIndex(sourceField) < 0) {
+ throw new HugeGraphConnectorException(
+ HugeGraphConnectorErrorCode.ILLEGAL_CONFIG_ARGUMENT,
+ String.format(
+ "Mapping[%s/%s]: Field '%s' configured in '%s' does not exist in input row",
+ mapping.getType(), mapping.getLabel(), sourceField, optionName));
+ }
+ }
+ }
}
diff --git a/seatunnel-connectors-v2/connector-hugegraph/src/test/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/buffer/BatchBufferTest.java b/seatunnel-connectors-v2/connector-hugegraph/src/test/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/buffer/BatchBufferTest.java
new file mode 100644
index 0000000000..44da78b37c
--- /dev/null
+++ b/seatunnel-connectors-v2/connector-hugegraph/src/test/java/org/apache/seatunnel/connectors/seatunnel/hugegraph/buffer/BatchBufferTest.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.seatunnel.hugegraph.buffer;
+
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.client.HugeGraphClient;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.config.MappingConfig.LabelType;
+import org.apache.seatunnel.connectors.seatunnel.hugegraph.exception.HugeGraphConnectorException;
+
+import org.apache.hugegraph.structure.graph.Edge;
+import org.apache.hugegraph.structure.graph.UpdateStrategy;
+import org.apache.hugegraph.structure.graph.Vertex;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.InOrder;
... 7245 lines suppressed ...