You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ossie.apache.org by kb...@apache.org on 2026/07/23 21:01:21 UTC
(ossie) branch main updated: Add datatype field to Field and Metric; reframe is_time as role marker (#113)
This is an automated email from the ASF dual-hosted git repository.
khush-bhatia pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ossie.git
The following commit(s) were added to refs/heads/main by this push:
new 6c31730 Add datatype field to Field and Metric; reframe is_time as role marker (#113)
6c31730 is described below
commit 6c3173054f2b0556bd7bedbb3f204ee5712f1600
Author: Jon Mease <jo...@gmail.com>
AuthorDate: Thu Jul 23 17:01:15 2026 -0400
Add datatype field to Field and Metric; reframe is_time as role marker (#113)
* Add datatype field to Field and Metric; reframe is_time as role marker
Introduces a top-level `datatype` on Field and Metric with a closed logical
enum: string, integer, number, boolean, date, time, timestamp, timestamp_tz,
other. Addresses issue #84.
`datatype` and `dimension.is_time` are independent and orthogonal:
- `datatype` declares the field's logical data type (casting/serialization).
- `is_time` is a temporal-role marker (time-series analysis, temporal
filtering). A field with `is_time: true` may carry any `datatype` (e.g.
integer for a year grain, string for a month name, date for a calendar
date).
When `is_time` is unset, it defaults to `true` for temporal datatypes
(`date`, `time`, `timestamp`, `timestamp_tz`) and `false` otherwise.
Explicit `is_time` always wins, so authors can set `is_time: false` on
an audit `created_at` to keep it off the time axis.
Taxonomy and type/role split were chosen after benchmarking 14 peer
semantic layers and 5 portable type standards. Notable precedent:
Snowflake Semantic Views' YAML authoring form has a `time_dimensions:`
collection whose entries can carry any `data_type` (the published example
annotates `order_year` with `data_type: NUMBER`); LookML's `dimension_group`
accepts `date`, `datetime`, `timestamp`, `epoch`, and `yyyymmdd`.
Snowflake converter updated: `_classify_field` honors explicit `is_time`
first, then falls back to the temporal-datatype default. 9 new tests
cover the datatype paths and the mixed-metadata cases (`d_year` with
`datatype: integer` and `is_time: true`, audit timestamp opt-out, etc.).
tpcds_semantic_model.yaml demonstrates three coexistence patterns:
datatype-only, datatype + is_time, and is_time-only.
Also added .gitignore for __pycache__ directories.
* Align DataType enum with ontology value types
* Add DataType to Python bindings
* Map Ossie DataType to Snowflake fields
* Map Ossie DataType in GoodData conversions
* Map Ossie DataType in Polaris conversions
* Map Ossie DataType in Salesforce conversions
* Consolidate datatype test coverage
* Fix GoodData datatype edge cases
* Honor effective time roles in dbt export
* Warn on lossy Salesforce DateTime export
* Define datatype enum in YAML spec
* Remove trailing whitespace from spec example
* Add syncing todo
---
.gitignore | 1 +
converters/README.md | 9 +-
converters/dbt/src/ossie_dbt/osi_to_msi.py | 4 +-
converters/dbt/tests/test_osi_to_msi.py | 24 ++
converters/gooddata/README.md | 28 +++
.../src/ossie_gooddata/datatype_mapping.py | 124 ++++++++++
.../gooddata/src/ossie_gooddata/gooddata_to_osi.py | 16 ++
converters/gooddata/src/ossie_gooddata/models.py | 12 +-
.../gooddata/src/ossie_gooddata/osi_to_gooddata.py | 33 ++-
converters/gooddata/tests/test_gooddata_to_osi.py | 111 ++++++++-
converters/gooddata/tests/test_models.py | 4 +
converters/gooddata/tests/test_osi_to_gooddata.py | 248 +++++++++++++++++++-
converters/gooddata/tests/test_roundtrip.py | 70 +++++-
converters/polaris/README.md | 53 ++++-
.../ossie/converter/polaris/IcebergTypeMapper.java | 252 +++++++++++++++++++++
.../ossie/converter/polaris/OsiModelParser.java | 30 ++-
.../ossie/converter/polaris/OsiYamlGenerator.java | 27 ++-
.../ossie/converter/polaris/PolarisExporter.java | 71 +++++-
.../ossie/converter/polaris/PolarisImporter.java | 62 ++---
.../ossie/converter/polaris/model/OsiModel.java | 8 +
.../converter/polaris/IcebergTypeMapperTest.java | 141 ++++++++++++
.../converter/polaris/OsiPolarisConverterTest.java | 188 +++++++++++----
converters/salesforce/README.md | 72 +++++-
.../apache/ossie/converter/ConverterConstants.java | 5 +-
.../ossie/converter/FieldMappingHandler.java | 61 ++++-
.../ossie/converter/MetricMappingHandler.java | 5 +
.../ossie/converter/SalesforceDataTypeMapper.java | 93 ++++++++
.../apache/ossie/OsiToSalesforceConverterTest.java | 18 ++
.../apache/ossie/SalesforceToOsiConverterTest.java | 22 ++
.../converter/SalesforceDataTypeMapperTest.java | 112 +++++++++
.../test/resources/examples/osiToSalesforce.yaml | 18 ++
converters/snowflake/README.md | 23 ++
.../snowflake/src/ossie_snowflake/converter.py | 68 +++++-
.../example_converted_tpcds_semantic_model.yaml | 29 +++
.../tests/test_osi_to_snowflake_yaml_converter.py | 108 +++++++++
core-spec/osi-schema.json | 24 +-
core-spec/spec.md | 65 +++++-
core-spec/spec.yaml | 44 +++-
docs/index.md | 2 +-
examples/tpcds_semantic_model.yaml | 41 +++-
python/src/ossie/__init__.py | 2 +
python/src/ossie/models.py | 40 ++++
python/tests/test_models.py | 137 +++++++++++
43 files changed, 2324 insertions(+), 181 deletions(-)
diff --git a/.gitignore b/.gitignore
index 3d183e1..9e63dcd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,6 +24,7 @@
**/htmlcov/
**/dist/
**/target/
+*.py[cod]
# Go CLI
cli/dist/
diff --git a/converters/README.md b/converters/README.md
index 48bc065..ac31d85 100644
--- a/converters/README.md
+++ b/converters/README.md
@@ -114,11 +114,17 @@ Datasets represent logical tables (fact or dimension tables). They contain field
Fields represent row-level attributes. They can be simple column references or computed expressions.
+> **Note:** `datatype` (on `Field` and `Metric`) declares a field's logical data type; `dimension.is_time` is an independent temporal-role marker.
+> A field may carry both, either, or neither. Use `datatype` for data-type questions (casting, serialization); use `is_time` for role questions
+> (classifying time dimensions). When `is_time` is unset it defaults to `true` if `datatype` is one of `Date`, `Time`, `DateTime`, `DateTimeTz`,
+> and `false` otherwise. Explicit `is_time` always wins.
+
| Ossie Field | Description | Converter Consideration |
|-----------|-------------|------------------------|
| `name` | Field identifier | Map to column/attribute name |
| `expression.dialects` | Multi-dialect SQL expressions | Select the dialect matching the target vendor; fall back to `ANSI_SQL` |
-| `dimension.is_time` | Whether the field is a time dimension | Map to vendor-specific time dimension markers |
+| `datatype` | Logical data type of the field (one of `String`, `Integer`, `Decimal`, `Float`, `Boolean`, `Date`, `Time`, `DateTime`, `DateTimeTz`, `Opaque`). | Converters SHOULD consult `datatype` for the field's logical data type. `Decimal` is exact base-10 with unspecified precision and scale; `Float` is approximate. Prefer the temporal members (`Date`, `Time`, `DateTime`, `DateTimeTz`) to classify time dimensions. Omit `datatype` when unknown; use `Opaque` + `custom_extensions` for a [...]
+| `dimension.is_time` | Temporal-role marker. When `true`, the field should be treated as a time dimension regardless of its `datatype` (e.g. an `Integer` year grain, a `String` month name, or a `Date` column). When unset, defaults to `true` for temporal `datatype`s (`Date`, `Time`, `DateTime`, `DateTimeTz`) and `false` otherwise. | Map to vendor-specific time dimension markers. Converters SHOULD classify as a time dimension when `is_time` resolves to `true` (either explicit or defaulted [...]
| `label` | Categorization label | Map if vendor supports field labels/tags |
| `description` | Human-readable description | Most vendors support field descriptions |
| `ai_context` | Synonyms and business context | Map if vendor supports semantic annotations |
@@ -165,6 +171,7 @@ Metrics are aggregate measures defined at the semantic model level. They can spa
|-----------|-------------|------------------------|
| `name` | Metric identifier | Map to vendor's measure/KPI name |
| `expression.dialects` | Multi-dialect aggregate expressions | Select the appropriate dialect; fall back to `ANSI_SQL` |
+| `datatype` | Logical data type of the metric result (one of `String`, `Integer`, `Decimal`, `Float`, `Boolean`, `Date`, `Time`, `DateTime`, `DateTimeTz`, `Opaque`). | Converters SHOULD consult `datatype` to declare the result type of the aggregation. Numeric measures should distinguish exact `Decimal` or `Integer` results from approximate `Float` results. Omit `datatype` when unknown; use `Opaque` + `custom_extensions` for a known type outside the portable vocabulary. |
| `description` | What the metric measures | Most vendors support descriptions |
| `ai_context` | Synonyms and business context | Map if vendor supports semantic annotations |
diff --git a/converters/dbt/src/ossie_dbt/osi_to_msi.py b/converters/dbt/src/ossie_dbt/osi_to_msi.py
index 1da337d..d6116fe 100644
--- a/converters/dbt/src/ossie_dbt/osi_to_msi.py
+++ b/converters/dbt/src/ossie_dbt/osi_to_msi.py
@@ -182,7 +182,7 @@ class OSIToMSIConverter:
1. primary_key → PRIMARY entity
2. unique_keys → UNIQUE entity
3. foreign key (from relationship) → FOREIGN entity
- 4. dimension.is_time → TIME dimension (granularity defaults to DAY)
+ 4. effective time-dimension role → TIME dimension (granularity defaults to DAY)
5. fallback → CATEGORICAL dimension
Aggregation info lives on metrics (`metric_aggregation_params`), not on
@@ -227,7 +227,7 @@ class OSIToMSIConverter:
)
)
return
- if field.dimension is not None and field.dimension.is_time:
+ if field.is_time_dimension():
# Ossie carries no granularity metadata; default to DAY.
dimensions.append(
PydanticDimension(
diff --git a/converters/dbt/tests/test_osi_to_msi.py b/converters/dbt/tests/test_osi_to_msi.py
index 9a2bf86..6f93a16 100644
--- a/converters/dbt/tests/test_osi_to_msi.py
+++ b/converters/dbt/tests/test_osi_to_msi.py
@@ -20,6 +20,7 @@
import pytest
from syrupy.assertion import SnapshotAssertion
+from ossie import OSIDataType, OSIDimension
from ossie_dbt.msi_to_osi import MSIToOSIConverter
from ossie_dbt.osi_to_msi import OSIToMSIConverter
from metricflow_semantic_interfaces.type_enums import (
@@ -156,6 +157,29 @@ class TestOSIToMSIFieldClassification:
assert sm.dimensions[0].name == field_name
assert sm.dimensions[0].type == expected_type
+ @pytest.mark.parametrize(
+ ("dimension", "datatype", "expected_type"),
+ [
+ (OSIDimension(), OSIDataType.DATE, DimensionType.TIME),
+ (OSIDimension(is_time=False), OSIDataType.DATE_TIME_TZ, DimensionType.CATEGORICAL),
+ (None, OSIDataType.DATE, DimensionType.CATEGORICAL),
+ ],
+ )
+ def test_field_becomes_dimension_by_effective_time_role(
+ self,
+ dimension: OSIDimension | None,
+ datatype: OSIDataType,
+ expected_type: DimensionType,
+ ) -> None:
+ field = _osi_field("occurred_at").model_copy(
+ update={"dimension": dimension, "datatype": datatype}
+ )
+ doc = _osi_doc(datasets=[_osi_dataset("events", fields=[field])])
+
+ sm = OSIToMSIConverter().convert(doc).output.semantic_models[0]
+
+ assert sm.dimensions[0].type == expected_type
+
def test_field_referenced_in_metric_stays_as_dimension(self) -> None:
"""Fields referenced in metric expressions are no longer promoted to measures."""
doc = _osi_doc(
diff --git a/converters/gooddata/README.md b/converters/gooddata/README.md
index 8262b1c..f238866 100644
--- a/converters/gooddata/README.md
+++ b/converters/gooddata/README.md
@@ -67,10 +67,32 @@ uv run pytest
| Dataset | Dataset |
| Attribute (+ Labels) | Field with `dimension` metadata |
| Fact | Field without `dimension` metadata |
+| Source column data type | Field `datatype` |
| Reference (FK) | Relationship |
| Date Instance | Dataset with `GOODDATA` custom_extension (`date_dimension: true`) |
| MAQL expression | Dialect entry (`dialect: MAQL`) |
+### Data Types
+
+| GoodData source type | Ossie `DataType` |
+|---|---|
+| `STRING` | `String` |
+| `INT` | `Integer` |
+| `NUMERIC` | `Decimal` |
+| `BOOLEAN` | `Boolean` |
+| `DATE` | `Date` |
+| `TIMESTAMP` | `DateTime` |
+| `TIMESTAMP_TZ` | `DateTimeTz` |
+
+Only an explicitly present GoodData `sourceColumnDataType` becomes an Ossie
+`datatype`. If GoodData omits the source type, the Ossie datatype remains
+unspecified rather than being inferred from whether the field is an attribute or fact.
+
+Unknown GoodData source types become `Opaque` and their exact value is retained in a
+`GOODDATA` custom extension for lossless round trips. Regular GoodData attributes remain
+explicitly non-time dimensions even when their source column has a temporal data type;
+GoodData date instances continue to carry the time-dimension role.
+
## Limitations
- **Metrics are not converted.** GoodData metrics use MAQL, a context-aware metric language
@@ -78,3 +100,9 @@ uv run pytest
is SQL-expression-based and cannot represent this paradigm.
- AggregatedFacts are not yet supported.
- Workspace data filters are not mapped.
+- GoodData has only one `NUMERIC` source type, so exporting Ossie `Float` loses the
+ distinction between approximate and exact numeric values.
+- GoodData has no time-only source type. Ossie `Time` fields retain the existing
+ attribute or fact default and emit a warning when exported.
+- Ossie `Opaque` fields require an exact GoodData source type in their `GOODDATA`
+ extension for lossless export; otherwise the role default is used with a warning.
diff --git a/converters/gooddata/src/ossie_gooddata/datatype_mapping.py b/converters/gooddata/src/ossie_gooddata/datatype_mapping.py
new file mode 100644
index 0000000..6f57b32
--- /dev/null
+++ b/converters/gooddata/src/ossie_gooddata/datatype_mapping.py
@@ -0,0 +1,124 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Data type mapping shared by the GoodData converters."""
+
+from __future__ import annotations
+
+import warnings
+
+GOODDATA_TO_OSSIE_DATATYPES = {
+ "STRING": "String",
+ "INT": "Integer",
+ "NUMERIC": "Decimal",
+ "BOOLEAN": "Boolean",
+ "DATE": "Date",
+ "TIMESTAMP": "DateTime",
+ "TIMESTAMP_TZ": "DateTimeTz",
+}
+
+OSSIE_TO_GOODDATA_DATATYPES = {value: key for key, value in GOODDATA_TO_OSSIE_DATATYPES.items()}
+
+def gooddata_to_ossie_datatype(source_type: str | None) -> str | None:
+ """Map a GoodData source column type to an Ossie DataType value."""
+ if source_type is None or not source_type.strip():
+ return None
+ return GOODDATA_TO_OSSIE_DATATYPES.get(source_type.strip().upper(), "Opaque")
+
+
+def ossie_to_gooddata_datatype(
+ datatype: str | None,
+ *,
+ default: str,
+ field_name: str,
+ extension_type: str | None = None,
+) -> str:
+ """Choose a GoodData source type, preserving exact extension data first."""
+ mapped_type = OSSIE_TO_GOODDATA_DATATYPES.get(datatype) if datatype is not None else None
+
+ if extension_type is not None and extension_type.strip():
+ normalized_extension_type = extension_type.strip().upper()
+ if datatype is None or datatype == "Opaque":
+ return normalized_extension_type
+ if mapped_type is not None and normalized_extension_type != mapped_type:
+ warnings.warn(
+ f"Field '{field_name}' has Ossie datatype '{datatype}', which maps to "
+ f"GoodData '{mapped_type}', but its GOODDATA extension specifies "
+ f"'{normalized_extension_type}'. Preserving the extension value.",
+ stacklevel=2,
+ )
+ elif datatype == "Float":
+ if normalized_extension_type == "NUMERIC":
+ warnings.warn(
+ f"Field '{field_name}' has Ossie datatype 'Float'; GoodData only has "
+ "NUMERIC, so the exact/approximate distinction will be lost.",
+ stacklevel=2,
+ )
+ else:
+ warnings.warn(
+ f"Field '{field_name}' has Ossie datatype 'Float', which maps lossily to "
+ f"GoodData 'NUMERIC', but its GOODDATA extension specifies "
+ f"'{normalized_extension_type}'. Preserving the extension value.",
+ stacklevel=2,
+ )
+ elif datatype == "Time":
+ warnings.warn(
+ f"Field '{field_name}' has Ossie datatype 'Time', which has no native "
+ f"GoodData source column type. Preserving the extension value "
+ f"'{normalized_extension_type}'.",
+ stacklevel=2,
+ )
+ elif mapped_type is None:
+ warnings.warn(
+ f"Field '{field_name}' has unrecognized Ossie datatype '{datatype}'. "
+ f"Preserving the GOODDATA extension value '{normalized_extension_type}'.",
+ stacklevel=2,
+ )
+ return normalized_extension_type
+
+ if datatype is None:
+ return default
+ if mapped_type is not None:
+ return mapped_type
+ if datatype == "Float":
+ warnings.warn(
+ f"Field '{field_name}' has Ossie datatype 'Float'; GoodData only has "
+ "NUMERIC, so the exact/approximate distinction will be lost.",
+ stacklevel=2,
+ )
+ return "NUMERIC"
+ if datatype == "Time":
+ warnings.warn(
+ f"Field '{field_name}' has Ossie datatype 'Time', which has no native "
+ f"GoodData source column type. Using the role default '{default}'.",
+ stacklevel=2,
+ )
+ return default
+ if datatype == "Opaque":
+ warnings.warn(
+ f"Field '{field_name}' has Ossie datatype 'Opaque' without an exact "
+ f"GoodData source type in its GOODDATA extension. Using the role default '{default}'.",
+ stacklevel=2,
+ )
+ return default
+
+ warnings.warn(
+ f"Field '{field_name}' has unrecognized Ossie datatype '{datatype}'. "
+ f"Using the GoodData role default '{default}'.",
+ stacklevel=2,
+ )
+ return default
diff --git a/converters/gooddata/src/ossie_gooddata/gooddata_to_osi.py b/converters/gooddata/src/ossie_gooddata/gooddata_to_osi.py
index db68133..c658d29 100644
--- a/converters/gooddata/src/ossie_gooddata/gooddata_to_osi.py
+++ b/converters/gooddata/src/ossie_gooddata/gooddata_to_osi.py
@@ -22,6 +22,7 @@ from __future__ import annotations
import json
from typing import Any
+from ossie_gooddata.datatype_mapping import gooddata_to_ossie_datatype
from ossie_gooddata.models import (
GdAttribute,
GdDataset,
@@ -162,6 +163,7 @@ def _convert_date_instance(di: GdDateInstance) -> dict[str, Any]:
def _convert_attribute(attr: GdAttribute, dataset_id: str) -> dict[str, Any]:
"""Convert a GoodData attribute to an Ossie field."""
+ datatype = gooddata_to_ossie_datatype(attr.source_column_data_type)
osi_field: dict[str, Any] = {
"name": attr.source_column,
"expression": {
@@ -172,6 +174,8 @@ def _convert_attribute(attr: GdAttribute, dataset_id: str) -> dict[str, Any]:
},
"dimension": {"is_time": False},
}
+ if datatype is not None:
+ osi_field["datatype"] = datatype
if attr.description:
osi_field["description"] = attr.description
if attr.title != attr.source_column:
@@ -179,6 +183,8 @@ def _convert_attribute(attr: GdAttribute, dataset_id: str) -> dict[str, Any]:
# Store GoodData-specific attribute metadata in custom_extensions
ext = _build_attribute_extension(attr)
+ if datatype == "Opaque":
+ ext["source_column_data_type"] = attr.source_column_data_type
if ext:
osi_field["custom_extensions"] = [{"vendor_name": "GOODDATA", "data": json.dumps(ext)}]
@@ -187,6 +193,7 @@ def _convert_attribute(attr: GdAttribute, dataset_id: str) -> dict[str, Any]:
def _convert_fact(fact: GdFact, dataset_id: str) -> dict[str, Any]:
"""Convert a GoodData fact to an Ossie field."""
+ datatype = gooddata_to_ossie_datatype(fact.source_column_data_type)
osi_field: dict[str, Any] = {
"name": fact.source_column,
"expression": {
@@ -196,10 +203,19 @@ def _convert_fact(fact: GdFact, dataset_id: str) -> dict[str, Any]:
],
},
}
+ if datatype is not None:
+ osi_field["datatype"] = datatype
if fact.description:
osi_field["description"] = fact.description
if fact.title != fact.source_column:
osi_field.setdefault("ai_context", {})["synonyms"] = [fact.title]
+ if datatype == "Opaque":
+ osi_field["custom_extensions"] = [
+ {
+ "vendor_name": "GOODDATA",
+ "data": json.dumps({"source_column_data_type": fact.source_column_data_type}),
+ }
+ ]
return osi_field
diff --git a/converters/gooddata/src/ossie_gooddata/models.py b/converters/gooddata/src/ossie_gooddata/models.py
index 2ef93b0..e96a8a6 100644
--- a/converters/gooddata/src/ossie_gooddata/models.py
+++ b/converters/gooddata/src/ossie_gooddata/models.py
@@ -67,7 +67,7 @@ class GdAttribute:
title: str
source_column: str
description: str = ""
- source_column_data_type: str = "STRING"
+ source_column_data_type: str | None = None
sort_column: str | None = None
sort_direction: str | None = None
labels: list[GdLabel] = field(default_factory=list)
@@ -80,7 +80,7 @@ class GdFact:
title: str
source_column: str
description: str = ""
- source_column_data_type: str = "NUMERIC"
+ source_column_data_type: str | None = None
tags: list[str] = field(default_factory=list)
@@ -192,7 +192,7 @@ def gd_model_from_dict(data: dict[str, Any]) -> GdDeclarativeModel:
title=attr.get("title", ""),
source_column=attr.get("sourceColumn", ""),
description=attr.get("description", ""),
- source_column_data_type=attr.get("sourceColumnDataType", "STRING"),
+ source_column_data_type=attr.get("sourceColumnDataType"),
sort_column=attr.get("sortColumn"),
sort_direction=attr.get("sortDirection"),
labels=labels,
@@ -206,7 +206,7 @@ def gd_model_from_dict(data: dict[str, Any]) -> GdDeclarativeModel:
title=f.get("title", ""),
source_column=f.get("sourceColumn", ""),
description=f.get("description", ""),
- source_column_data_type=f.get("sourceColumnDataType", "NUMERIC"),
+ source_column_data_type=f.get("sourceColumnDataType"),
tags=f.get("tags", []),
)
for f in ds.get("facts", [])
@@ -366,7 +366,7 @@ def _attr_to_dict(a: GdAttribute) -> dict[str, Any]:
}
if a.description:
d["description"] = a.description
- if a.source_column_data_type != "STRING":
+ if a.source_column_data_type is not None and a.source_column_data_type != "STRING":
d["sourceColumnDataType"] = a.source_column_data_type
if a.sort_column:
d["sortColumn"] = a.sort_column
@@ -385,7 +385,7 @@ def _fact_to_dict(f: GdFact) -> dict[str, Any]:
}
if f.description:
d["description"] = f.description
- if f.source_column_data_type != "NUMERIC":
+ if f.source_column_data_type is not None and f.source_column_data_type != "NUMERIC":
d["sourceColumnDataType"] = f.source_column_data_type
if f.tags:
d["tags"] = f.tags
diff --git a/converters/gooddata/src/ossie_gooddata/osi_to_gooddata.py b/converters/gooddata/src/ossie_gooddata/osi_to_gooddata.py
index f7f05c8..a8dbac1 100644
--- a/converters/gooddata/src/ossie_gooddata/osi_to_gooddata.py
+++ b/converters/gooddata/src/ossie_gooddata/osi_to_gooddata.py
@@ -23,6 +23,7 @@ import json
import re
from typing import Any
+from ossie_gooddata.datatype_mapping import ossie_to_gooddata_datatype
from ossie_gooddata.models import (
GdAttribute,
GdDataset,
@@ -99,8 +100,16 @@ def _is_date_dataset(ds: dict[str, Any]) -> bool:
gd_ext = _get_gooddata_extension(ds)
if gd_ext and gd_ext.get("date_dimension"):
return True
+ return _is_legacy_date_dataset(ds, gd_ext)
+
+
+def _is_legacy_date_dataset(
+ ds: dict[str, Any],
+ gd_ext: dict[str, Any] | None = None,
+) -> bool:
+ """Detect the converter's legacy all-explicit-time date-dataset shape."""
fields = ds.get("fields", [])
- return bool(fields) and all(_is_time_field(f) for f in fields) and not gd_ext
+ return bool(fields) and all(_has_explicit_time_role(f) for f in fields) and not gd_ext
def _build_relationship_map(
@@ -129,10 +138,11 @@ def _convert_osi_dataset(
if gd_ext and gd_ext.get("date_dimension"):
return _placeholder_dataset(ds_name), _convert_to_date_instance(ds, gd_ext)
- # Check if all fields are time dimensions — heuristic for date datasets
+ # Preserve the legacy explicit-is_time heuristic for date datasets. The
+ # general temporal datatype default does not imply this GoodData-specific
+ # dataset kind.
fields = ds.get("fields", [])
- all_time = fields and all(_is_time_field(f) for f in fields)
- if all_time and not gd_ext:
+ if _is_legacy_date_dataset(ds, gd_ext):
return _placeholder_dataset(ds_name), _convert_to_date_instance_from_fields(ds)
# Regular dataset
@@ -234,6 +244,12 @@ def _convert_to_attribute(field_def: dict[str, Any], dataset_id: str) -> GdAttri
title=_get_title(field_def, fallback=field_name),
source_column=source_col,
description=field_def.get("description", ""),
+ source_column_data_type=ossie_to_gooddata_datatype(
+ field_def.get("datatype"),
+ default="STRING",
+ field_name=field_name,
+ extension_type=gd_ext.get("source_column_data_type") if gd_ext else None,
+ ),
sort_column=gd_ext.get("sort_column") if gd_ext else None,
sort_direction=gd_ext.get("sort_direction") if gd_ext else None,
labels=labels,
@@ -244,12 +260,19 @@ def _convert_to_fact(field_def: dict[str, Any], dataset_id: str) -> GdFact:
"""Convert an Ossie field (non-dimension) to a GoodData fact."""
field_name = field_def["name"]
source_col = _get_source_column(field_def)
+ gd_ext = _get_gooddata_extension(field_def)
return GdFact(
id=f"fact.{dataset_id}.{field_name}",
title=_get_title(field_def, fallback=field_name),
source_column=source_col,
description=field_def.get("description", ""),
+ source_column_data_type=ossie_to_gooddata_datatype(
+ field_def.get("datatype"),
+ default="NUMERIC",
+ field_name=field_name,
+ extension_type=gd_ext.get("source_column_data_type") if gd_ext else None,
+ ),
)
@@ -294,7 +317,7 @@ def _get_title(obj: dict[str, Any], fallback: str = "") -> str:
return obj.get("description", "") or obj.get("name", "") or fallback
-def _is_time_field(field_def: dict[str, Any]) -> bool:
+def _has_explicit_time_role(field_def: dict[str, Any]) -> bool:
dim = field_def.get("dimension")
return isinstance(dim, dict) and dim.get("is_time") is True
diff --git a/converters/gooddata/tests/test_gooddata_to_osi.py b/converters/gooddata/tests/test_gooddata_to_osi.py
index 637f77d..f330ba3 100644
--- a/converters/gooddata/tests/test_gooddata_to_osi.py
+++ b/converters/gooddata/tests/test_gooddata_to_osi.py
@@ -19,8 +19,99 @@
from __future__ import annotations
-from ossie_gooddata.gooddata_to_osi import gooddata_to_osi
-from ossie_gooddata.models import GdDeclarativeModel
+import json
+
+import pytest
+
+from ossie_gooddata.gooddata_to_osi import (
+ _convert_attribute,
+ _convert_fact,
+ gooddata_to_osi,
+)
+from ossie_gooddata.models import GdAttribute, GdDeclarativeModel, GdFact
+
+
+@pytest.mark.parametrize(
+ ("gooddata_type", "ossie_type"),
+ [
+ ("STRING", "String"),
+ ("INT", "Integer"),
+ ("NUMERIC", "Decimal"),
+ ("BOOLEAN", "Boolean"),
+ ("DATE", "Date"),
+ ("TIMESTAMP", "DateTime"),
+ ("TIMESTAMP_TZ", "DateTimeTz"),
+ ],
+)
+def test_native_source_types_become_ossie_datatypes(gooddata_type: str, ossie_type: str):
+ """Verify native GoodData types map on both attributes and facts."""
+ attribute = GdAttribute(
+ id="attr.orders.value",
+ title="Value",
+ source_column="value",
+ source_column_data_type=gooddata_type,
+ )
+ fact = GdFact(
+ id="fact.orders.value",
+ title="Value",
+ source_column="value",
+ source_column_data_type=gooddata_type,
+ )
+
+ osi_attribute = _convert_attribute(attribute, "orders")
+ osi_fact = _convert_fact(fact, "orders")
+
+ assert osi_attribute["datatype"] == ossie_type
+ assert osi_attribute["dimension"]["is_time"] is False
+ assert osi_fact["datatype"] == ossie_type
+ assert "dimension" not in osi_fact
+
+
+@pytest.mark.parametrize(
+ ("source_type", "converter", "model"),
+ [
+ (
+ "CUSTOM_ATTRIBUTE_TYPE",
+ _convert_attribute,
+ GdAttribute(
+ id="attr.orders.value",
+ title="Value",
+ source_column="value",
+ source_column_data_type="CUSTOM_ATTRIBUTE_TYPE",
+ ),
+ ),
+ (
+ "CUSTOM_FACT_TYPE",
+ _convert_fact,
+ GdFact(
+ id="fact.orders.value",
+ title="Value",
+ source_column="value",
+ source_column_data_type="CUSTOM_FACT_TYPE",
+ ),
+ ),
+ ],
+)
+def test_unknown_source_type_becomes_opaque_and_is_preserved(source_type: str, converter, model):
+ """Verify unknown GoodData types survive in an exact vendor extension."""
+ osi_field = converter(model, "orders")
+
+ assert osi_field["datatype"] == "Opaque"
+ extension = json.loads(osi_field["custom_extensions"][0]["data"])
+ assert extension["source_column_data_type"] == source_type
+
+
+@pytest.mark.parametrize("source_type", [None, ""])
+def test_missing_or_empty_source_type_is_omitted(source_type: str | None):
+ """Verify a missing source type does not invent an Ossie datatype."""
+ attribute = GdAttribute(
+ id="attr.orders.value",
+ title="Value",
+ source_column="value",
+ source_column_data_type=source_type,
+ )
+
+ assert "datatype" not in _convert_attribute(attribute, "orders")
def test_basic_conversion(gooddata_tpcds_model: GdDeclarativeModel):
@@ -100,6 +191,19 @@ def test_facts_become_plain_fields(gooddata_tpcds_model: GdDeclarativeModel):
assert len(fact_fields) == 4
+def test_omitted_fixture_source_types_remain_unspecified(gooddata_tpcds_model: GdDeclarativeModel):
+ """Verify absent GoodData source types are not promoted from field roles."""
+ result = gooddata_to_osi(gooddata_tpcds_model)
+ store_sales = next(
+ ds for ds in result["semantic_model"][0]["datasets"] if ds["name"] == "store_sales"
+ )
+
+ item_key = next(field for field in store_sales["fields"] if field["name"] == "ss_item_sk")
+ quantity = next(field for field in store_sales["fields"] if field["name"] == "ss_quantity")
+ assert "datatype" not in item_key
+ assert "datatype" not in quantity
+
+
def test_maql_expressions(gooddata_tpcds_model: GdDeclarativeModel):
"""Verify MAQL dialect expressions are generated for fields."""
result = gooddata_to_osi(gooddata_tpcds_model)
@@ -141,7 +245,6 @@ def test_date_instance_converted(gooddata_tpcds_model: GdDeclarativeModel):
ext = date_ds["custom_extensions"][0]
assert ext["vendor_name"] == "GOODDATA"
- import json
ext_data = json.loads(ext["data"])
assert ext_data["date_dimension"] is True
@@ -158,7 +261,6 @@ def test_labels_in_custom_extensions(gooddata_tpcds_model: GdDeclarativeModel):
sk_field = next(f for f in customer["fields"] if f["name"] == "c_customer_sk")
assert "custom_extensions" in sk_field
- import json
ext_data = json.loads(sk_field["custom_extensions"][0]["data"])
assert ext_data["field_type"] == "attribute"
@@ -174,7 +276,6 @@ def test_data_source_id_extension(gooddata_tpcds_model: GdDeclarativeModel):
sm = result["semantic_model"][0]
assert "custom_extensions" in sm
- import json
ext_data = json.loads(sm["custom_extensions"][0]["data"])
assert ext_data["data_source_id"] == "my_pg"
diff --git a/converters/gooddata/tests/test_models.py b/converters/gooddata/tests/test_models.py
index 64b67d7..f583c6f 100644
--- a/converters/gooddata/tests/test_models.py
+++ b/converters/gooddata/tests/test_models.py
@@ -36,6 +36,8 @@ def test_parse_gooddata_model(gooddata_tpcds_dict: dict, gooddata_tpcds_model: G
assert len(store_sales.facts) == 4
assert len(store_sales.grain) == 2
assert len(store_sales.references) == 4
+ assert store_sales.attributes[0].source_column_data_type is None
+ assert store_sales.facts[0].source_column_data_type is None
# Check data source table id
assert store_sales.data_source_table_id is not None
@@ -70,6 +72,8 @@ def test_roundtrip_serialization(gooddata_tpcds_dict: dict):
assert len(ds["attributes"]) == 4
assert len(ds["facts"]) == 4
assert len(ds["references"]) == 4
+ assert "sourceColumnDataType" not in ds["attributes"][0]
+ assert "sourceColumnDataType" not in ds["facts"][0]
# Check date instance preserved
di = result["ldm"]["dateInstances"][0]
diff --git a/converters/gooddata/tests/test_osi_to_gooddata.py b/converters/gooddata/tests/test_osi_to_gooddata.py
index abd4091..7848eec 100644
--- a/converters/gooddata/tests/test_osi_to_gooddata.py
+++ b/converters/gooddata/tests/test_osi_to_gooddata.py
@@ -19,7 +19,253 @@
from __future__ import annotations
-from ossie_gooddata.osi_to_gooddata import osi_to_gooddata
+import warnings
+
+import pytest
+
+from ossie_gooddata.osi_to_gooddata import (
+ _convert_to_attribute,
+ _convert_to_fact,
+ osi_to_gooddata,
+)
+
+
+def _field_with_extension(datatype: str | None, extension_type: str) -> dict:
+ field = {
+ "name": "value",
+ "custom_extensions": [
+ {
+ "vendor_name": "GOODDATA",
+ "data": {"source_column_data_type": extension_type},
+ }
+ ],
+ }
+ if datatype is not None:
+ field["datatype"] = datatype
+ return field
+
+
+def _direct_field(name: str, **properties) -> dict:
+ return {
+ "name": name,
+ "expression": {
+ "dialects": [{"dialect": "ANSI_SQL", "expression": name}],
+ },
+ **properties,
+ }
+
+
+@pytest.mark.parametrize(
+ ("ossie_type", "gooddata_type"),
+ [
+ ("String", "STRING"),
+ ("Integer", "INT"),
+ ("Decimal", "NUMERIC"),
+ ("Boolean", "BOOLEAN"),
+ ("Date", "DATE"),
+ ("DateTime", "TIMESTAMP"),
+ ("DateTimeTz", "TIMESTAMP_TZ"),
+ ],
+)
+def test_ossie_datatypes_become_native_source_types(ossie_type: str, gooddata_type: str):
+ """Verify portable Ossie types map on both attributes and facts."""
+ field = {"name": "value", "datatype": ossie_type}
+
+ assert _convert_to_attribute(field, "orders").source_column_data_type == gooddata_type
+ assert _convert_to_fact(field, "orders").source_column_data_type == gooddata_type
+
+
+def test_float_becomes_numeric_with_loss_warning():
+ """Verify approximate Float values use GoodData's single numeric type."""
+ with pytest.warns(UserWarning, match="exact/approximate distinction"):
+ fact = _convert_to_fact({"name": "value", "datatype": "Float"}, "orders")
+
+ assert fact.source_column_data_type == "NUMERIC"
+
+
+@pytest.mark.parametrize(
+ ("converter", "default"),
+ [(_convert_to_attribute, "STRING"), (_convert_to_fact, "NUMERIC")],
+)
+def test_time_uses_role_default_with_warning(converter, default: str):
+ """Verify time-only fields retain the converter's existing role default."""
+ with pytest.warns(UserWarning, match="no native GoodData source column type"):
+ converted = converter({"name": "value", "datatype": "Time"}, "orders")
+
+ assert converted.source_column_data_type == default
+
+
+def test_opaque_restores_exact_gooddata_extension_type():
+ """Verify Opaque can round-trip an otherwise unknown GoodData type."""
+ field = {
+ "name": "value",
+ "datatype": "Opaque",
+ "custom_extensions": [
+ {
+ "vendor_name": "GOODDATA",
+ "data": '{"source_column_data_type": "CUSTOM_TYPE"}',
+ }
+ ],
+ }
+
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ fact = _convert_to_fact(field, "orders")
+
+ assert fact.source_column_data_type == "CUSTOM_TYPE"
+
+
+def test_opaque_without_extension_uses_role_default_with_warning():
+ """Verify Opaque does not fabricate a native source type."""
+ with pytest.warns(UserWarning, match="without an exact GoodData source type"):
+ attribute = _convert_to_attribute({"name": "value", "datatype": "Opaque"}, "orders")
+
+ assert attribute.source_column_data_type == "STRING"
+
+
+def test_extension_type_takes_precedence_over_portable_mapping():
+ """Verify an exact GoodData extension wins, with a conflict warning."""
+ field = _field_with_extension("Integer", "CUSTOM_TYPE")
+
+ with pytest.warns(UserWarning, match="Preserving the extension value"):
+ attribute = _convert_to_attribute(field, "orders")
+
+ assert attribute.source_column_data_type == "CUSTOM_TYPE"
+
+
+@pytest.mark.parametrize(
+ ("datatype", "extension_type", "expected_type", "warning_pattern"),
+ [
+ ("Float", "BOOLEAN", "BOOLEAN", "maps lossily.*NUMERIC"),
+ ("Float", "NUMERIC", "NUMERIC", "exact/approximate distinction"),
+ ("Time", "TIMESTAMP", "TIMESTAMP", "no native GoodData source column type"),
+ ("FutureOssieType", "STRING", "STRING", "unrecognized Ossie datatype"),
+ ],
+)
+def test_extension_warning_for_lossy_or_conflicting_portable_types(
+ datatype: str,
+ extension_type: str,
+ expected_type: str,
+ warning_pattern: str,
+):
+ """Verify exact extension precedence does not hide portable type loss."""
+ with pytest.warns(UserWarning, match=warning_pattern) as caught:
+ attribute = _convert_to_attribute(
+ _field_with_extension(datatype, extension_type),
+ "orders",
+ )
+
+ assert len(caught) == 1
+ assert attribute.source_column_data_type == expected_type
+
+
+@pytest.mark.parametrize(
+ ("datatype", "extension_type", "expected_type"),
+ [
+ ("Integer", " int ", "INT"),
+ ("String", "string", "STRING"),
+ ("Opaque", "custom_type", "CUSTOM_TYPE"),
+ (None, " boolean ", "BOOLEAN"),
+ ],
+)
+def test_extension_type_is_normalized_without_spurious_warning(
+ datatype: str | None,
+ extension_type: str,
+ expected_type: str,
+):
+ """Verify extension values are canonicalized before GoodData emission."""
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ attribute = _convert_to_attribute(
+ _field_with_extension(datatype, extension_type),
+ "orders",
+ )
+
+ assert attribute.source_column_data_type == expected_type
+
+
+@pytest.mark.parametrize("converter", [_convert_to_attribute, _convert_to_fact])
+def test_omitted_datatype_keeps_role_default_without_warning(converter):
+ """Verify models without datatype retain the converter's old behavior."""
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ converter({"name": "value"}, "orders")
+
+
+def test_temporal_datatype_does_not_turn_regular_dataset_into_date_instance():
+ """Verify a temporal role does not imply GoodData's special date dataset kind."""
+ model = {
+ "semantic_model": [
+ {
+ "name": "events",
+ "datasets": [
+ {
+ "name": "event_times",
+ "source": "analytics.event_times",
+ "fields": [
+ _direct_field(
+ "occurred_at",
+ datatype="DateTimeTz",
+ dimension={},
+ description="Event occurrence time",
+ )
+ ],
+ },
+ {
+ "name": "observations",
+ "source": "analytics.observations",
+ "fields": [_direct_field("event_time", dimension={})],
+ },
+ ],
+ "relationships": [
+ {
+ "name": "observation_event_time",
+ "from": "observations",
+ "to": "event_times",
+ "from_columns": ["event_time"],
+ "to_columns": ["occurred_at"],
+ }
+ ],
+ }
+ ]
+ }
+
+ result = osi_to_gooddata(model)
+
+ assert result.ldm.date_instances == []
+ event_times = next(ds for ds in result.ldm.datasets if ds.id == "event_times")
+ assert len(event_times.attributes) == 1
+ assert event_times.attributes[0].description == "Event occurrence time"
+ assert event_times.attributes[0].source_column_data_type == "TIMESTAMP_TZ"
+ observations = next(ds for ds in result.ldm.datasets if ds.id == "observations")
+ target = observations.references[0].sources[0].target
+ assert target.type == "attribute"
+ assert target.id == "attr.event_times.occurred_at"
+
+
+def test_legacy_all_explicit_time_fields_still_create_date_instance():
+ """Verify the converter's pre-datatype date-dataset heuristic remains compatible."""
+ model = {
+ "semantic_model": [
+ {
+ "name": "calendar",
+ "datasets": [
+ {
+ "name": "calendar_dates",
+ "source": "analytics.calendar_dates",
+ "fields": [
+ _direct_field("date_label", dimension={"is_time": True})
+ ],
+ }
+ ],
+ }
+ ]
+ }
+
+ result = osi_to_gooddata(model)
+
+ assert result.ldm.datasets == []
+ assert [date_instance.id for date_instance in result.ldm.date_instances] == ["calendar_dates"]
def test_basic_conversion(osi_tpcds_dict: dict):
diff --git a/converters/gooddata/tests/test_roundtrip.py b/converters/gooddata/tests/test_roundtrip.py
index 89f4fe0..95b8a39 100644
--- a/converters/gooddata/tests/test_roundtrip.py
+++ b/converters/gooddata/tests/test_roundtrip.py
@@ -19,11 +19,79 @@
from __future__ import annotations
+import json
+
+import pytest
+
from ossie_gooddata.gooddata_to_osi import gooddata_to_osi
-from ossie_gooddata.models import GdDeclarativeModel
+from ossie_gooddata.models import (
+ GdAttribute,
+ GdDataset,
+ GdDeclarativeModel,
+ GdLdm,
+ gd_model_to_dict,
+)
from ossie_gooddata.osi_to_gooddata import osi_to_gooddata
+def _model_with_attribute(source_type: str | None) -> GdDeclarativeModel:
+ return GdDeclarativeModel(
+ ldm=GdLdm(
+ datasets=[
+ GdDataset(
+ id="orders",
+ title="Orders",
+ attributes=[
+ GdAttribute(
+ id="attr.orders.value",
+ title="Value",
+ source_column="value",
+ source_column_data_type=source_type,
+ )
+ ],
+ )
+ ]
+ )
+ )
+
+
+@pytest.mark.parametrize(
+ "source_type",
+ ["STRING", "INT", "NUMERIC", "BOOLEAN", "DATE", "TIMESTAMP", "TIMESTAMP_TZ"],
+)
+def test_roundtrip_preserves_native_source_types(source_type: str):
+ """Verify every directly mapped GoodData source type round-trips."""
+ ossie = gooddata_to_osi(_model_with_attribute(source_type))
+ result = osi_to_gooddata(ossie)
+
+ assert result.ldm.datasets[0].attributes[0].source_column_data_type == source_type
+
+
+def test_roundtrip_preserves_unknown_source_type_through_opaque():
+ """Verify an unknown GoodData type round-trips through Opaque extension data."""
+ ossie = gooddata_to_osi(_model_with_attribute("CUSTOM_TYPE"))
+ field = ossie["semantic_model"][0]["datasets"][0]["fields"][0]
+
+ assert field["datatype"] == "Opaque"
+ assert json.loads(field["custom_extensions"][0]["data"])["source_column_data_type"] == "CUSTOM_TYPE"
+
+ result = osi_to_gooddata(ossie)
+ assert result.ldm.datasets[0].attributes[0].source_column_data_type == "CUSTOM_TYPE"
+
+
+def test_roundtrip_keeps_missing_source_type_unasserted():
+ """Verify a missing input type stays absent in Ossie and serialized GoodData."""
+ model = _model_with_attribute(None)
+
+ ossie = gooddata_to_osi(model)
+ field = ossie["semantic_model"][0]["datasets"][0]["fields"][0]
+ result = gd_model_to_dict(osi_to_gooddata(ossie))
+ attribute = result["ldm"]["datasets"][0]["attributes"][0]
+
+ assert "datatype" not in field
+ assert "sourceColumnDataType" not in attribute
+
+
def test_roundtrip_preserves_datasets(gooddata_tpcds_model: GdDeclarativeModel):
"""Verify GoodData → Ossie → GoodData preserves dataset count and IDs."""
# GoodData -> Ossie
diff --git a/converters/polaris/README.md b/converters/polaris/README.md
index d2f252f..7aedbca 100644
--- a/converters/polaris/README.md
+++ b/converters/polaris/README.md
@@ -83,9 +83,10 @@ Each Ossie semantic model becomes a Polaris namespace, and each dataset becomes
| Namespace | `semantic_model` (name, description) |
| Table | `dataset` (name) |
| Table location (`catalog.namespace.table`) | `dataset.source` |
-| Schema fields | `field` with `ANSI_SQL` dialect expression |
+| Schema fields | `field` with `ANSI_SQL` dialect expression and logical `datatype` |
| `identifier-field-ids` | `dataset.primary_key` |
| Temporal types (`timestamp`, `timestamptz`, `date`, `time`) | `field.dimension.is_time: true` |
+| Exact Iceberg type JSON | `field.custom_extensions` (vendor: `POLARIS`) |
| Table properties | `dataset.custom_extensions` (vendor: `COMMON`) |
### Export (Apache Ossie → Polaris)
@@ -96,18 +97,48 @@ Each Ossie semantic model becomes a Polaris namespace, and each dataset becomes
| `dataset` | Table |
| `dataset.source` | Stored in table property `osi.source` |
| `dataset.primary_key` | `identifier-field-ids` |
-| `field` | Schema column |
-| `field.dimension.is_time: true` | `timestamptz` type |
+| `field.datatype` | Schema column type |
+| Untyped `field.dimension.is_time: true` | `timestamptz` fallback type |
| `dataset.description` | Table property `comment` |
-### Type Inference (Export)
-
-Since Ossie fields are expression-based and don't carry explicit types, the exporter infers Iceberg types using:
-
-1. **Round-trip hints** — if a field description starts with `Iceberg type:` (produced by the importer), that type is used directly.
-2. **Time dimensions** — fields with `dimension.is_time: true` map to `timestamptz`.
-3. **Name conventions** — `*_id` → `long`, `*_date` → `date`, `*_at`/`*_timestamp` → `timestamptz`, `*_amount`/`*_price` → `decimal(18,2)`, `*_count`/`quantity` → `int`, `is_*`/`has_*` → `boolean`.
-4. **Default** — `string`.
+### Data Types
+
+The importer maps Iceberg physical types to Ossie logical types as follows:
+
+| Iceberg type | Ossie `datatype` |
+|---|---|
+| `boolean` | `Boolean` |
+| `int`, `long` | `Integer` |
+| `float`, `double` | `Float` |
+| `decimal(P,S)` | `Decimal` |
+| `date` | `Date` |
+| `time` | `Time` |
+| `timestamp`, `timestamp_ns` | `DateTime` |
+| `timestamptz`, `timestamptz_ns` | `DateTimeTz` |
+| `string` | `String` |
+| `unknown` | omitted |
+| UUID, binary, fixed, variant, spatial, and nested types | `Opaque` |
+
+Because logical types omit physical details such as integer width, decimal precision,
+timestamp precision, and nested structure, the importer also stores every exact Iceberg
+type in a field-level `POLARIS` custom extension. Nested field IDs are regenerated when
+creating a new table so the resulting Iceberg schema remains valid.
+
+The exporter resolves types in this order:
+
+1. **Exact extension type** — restores the complete Iceberg type from a `POLARIS` extension.
+2. **Ossie `datatype`** — maps portable logical types to Iceberg defaults.
+3. **Legacy description hint** — restores `Iceberg type:` descriptions from older converter output.
+4. **Time role** — an untyped field with `dimension.is_time: true` maps to `timestamptz`.
+5. **Name conventions** — retains the previous `*_id`, `*_date`, numeric, boolean, and other heuristics.
+6. **Default** — `string`.
+
+`datatype` and `dimension.is_time` are independent: an explicitly typed `String` time
+dimension remains an Iceberg `string`, while an explicitly typed `DateTime` field becomes
+`timestamp` even when it is not assigned a time-dimension role. A `Decimal` without an
+exact extension uses `decimal(18,2)` with a warning because Ossie does not specify precision
+or scale. `Opaque` requires an exact extension for lossless export; otherwise legacy
+inference is used with a warning.
## Architecture
diff --git a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/IcebergTypeMapper.java b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/IcebergTypeMapper.java
new file mode 100644
index 0000000..48f82ef
--- /dev/null
+++ b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/IcebergTypeMapper.java
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ossie.converter.polaris;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.TextNode;
+import org.apache.ossie.converter.polaris.model.OsiModel.CustomExtension;
+import org.apache.ossie.converter.polaris.model.OsiModel.Field;
+
+import java.io.IOException;
+import java.util.Locale;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/** Shared Iceberg physical type and Ossie logical datatype mapping. */
+final class IcebergTypeMapper {
+
+ static final String POLARIS_VENDOR = "POLARIS";
+ static final String ICEBERG_TYPE_KEY = "iceberg_type";
+
+ private IcebergTypeMapper() {}
+
+ /** Map an Iceberg type JSON value to the portable Ossie datatype vocabulary. */
+ static String toOssieDatatype(JsonNode icebergType) {
+ String baseType = baseType(icebergType);
+ if (baseType == null) {
+ return null;
+ }
+ switch (baseType) {
+ case "boolean":
+ return "Boolean";
+ case "int":
+ case "long":
+ return "Integer";
+ case "float":
+ case "double":
+ return "Float";
+ case "decimal":
+ return "Decimal";
+ case "date":
+ return "Date";
+ case "time":
+ return "Time";
+ case "timestamp":
+ case "timestamp_ns":
+ return "DateTime";
+ case "timestamptz":
+ case "timestamptz_ns":
+ return "DateTimeTz";
+ case "string":
+ return "String";
+ case "unknown":
+ return null;
+ default:
+ return "Opaque";
+ }
+ }
+
+ /** Return the default Iceberg physical type for a portable Ossie datatype. */
+ static JsonNode toDefaultIcebergType(String datatype) {
+ if (datatype == null) {
+ return null;
+ }
+ switch (datatype) {
+ case "String":
+ return TextNode.valueOf("string");
+ case "Integer":
+ return TextNode.valueOf("long");
+ case "Decimal":
+ return TextNode.valueOf("decimal(18, 2)");
+ case "Float":
+ return TextNode.valueOf("double");
+ case "Boolean":
+ return TextNode.valueOf("boolean");
+ case "Date":
+ return TextNode.valueOf("date");
+ case "Time":
+ return TextNode.valueOf("time");
+ case "DateTime":
+ return TextNode.valueOf("timestamp");
+ case "DateTimeTz":
+ return TextNode.valueOf("timestamptz");
+ default:
+ return null;
+ }
+ }
+
+ static boolean isTemporalDatatype(String datatype) {
+ return "Date".equals(datatype)
+ || "Time".equals(datatype)
+ || "DateTime".equals(datatype)
+ || "DateTimeTz".equals(datatype);
+ }
+
+ /** Read the exact Iceberg type JSON stored by the Polaris importer. */
+ static JsonNode exactIcebergType(Field field, ObjectMapper objectMapper) {
+ for (CustomExtension extension : field.getCustomExtensions()) {
+ if (!POLARIS_VENDOR.equals(extension.getVendorName())) {
+ continue;
+ }
+ try {
+ JsonNode data = objectMapper.readTree(extension.getData());
+ JsonNode icebergType = data == null ? null : data.get(ICEBERG_TYPE_KEY);
+ if (icebergType == null) {
+ continue;
+ }
+ if (!icebergType.isTextual() && !icebergType.isObject()) {
+ warn(field.getName(), "ignoring a POLARIS iceberg_type that is not a string or object");
+ continue;
+ }
+ return icebergType;
+ } catch (IOException | RuntimeException e) {
+ warn(field.getName(), "ignoring invalid POLARIS extension data: " + e.getMessage());
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Clone an exact Iceberg type for a newly-created schema and assign fresh IDs
+ * to every nested struct field, list element, and map key/value.
+ */
+ static JsonNode prepareExactTypeForExport(JsonNode icebergType, AtomicInteger nextNestedId) {
+ if (icebergType == null || icebergType.isNull()) {
+ return null;
+ }
+ if (!icebergType.isObject()) {
+ return icebergType.deepCopy();
+ }
+
+ String baseType = baseType(icebergType);
+ // Accept object-shaped decimal/fixed values from older or non-standard producers,
+ // but emit the canonical Iceberg JSON string representation.
+ if ("decimal".equals(baseType) || "fixed".equals(baseType)) {
+ return TextNode.valueOf(displayIcebergType(icebergType));
+ }
+
+ ObjectNode copy = ((ObjectNode) icebergType).deepCopy();
+ if ("struct".equals(baseType)) {
+ JsonNode fieldsNode = copy.get("fields");
+ if (fieldsNode instanceof ArrayNode) {
+ for (JsonNode fieldNode : fieldsNode) {
+ if (fieldNode instanceof ObjectNode) {
+ ObjectNode nestedField = (ObjectNode) fieldNode;
+ nestedField.put("id", nextNestedId.getAndIncrement());
+ if (nestedField.has("type")) {
+ nestedField.set(
+ "type",
+ prepareExactTypeForExport(nestedField.get("type"), nextNestedId));
+ }
+ }
+ }
+ }
+ } else if ("list".equals(baseType)) {
+ copy.put("element-id", nextNestedId.getAndIncrement());
+ if (copy.has("element")) {
+ copy.set("element", prepareExactTypeForExport(copy.get("element"), nextNestedId));
+ }
+ } else if ("map".equals(baseType)) {
+ copy.put("key-id", nextNestedId.getAndIncrement());
+ if (copy.has("key")) {
+ copy.set("key", prepareExactTypeForExport(copy.get("key"), nextNestedId));
+ }
+ copy.put("value-id", nextNestedId.getAndIncrement());
+ if (copy.has("value")) {
+ copy.set("value", prepareExactTypeForExport(copy.get("value"), nextNestedId));
+ }
+ }
+ return copy;
+ }
+
+ /** Human-readable Iceberg type used by the converter's legacy descriptions. */
+ static String displayIcebergType(JsonNode icebergType) {
+ if (icebergType == null || icebergType.isNull()) {
+ return "unknown";
+ }
+ if (icebergType.isTextual()) {
+ return icebergType.asText();
+ }
+ if (!icebergType.isObject()) {
+ return "unknown";
+ }
+
+ String type = baseType(icebergType);
+ if ("list".equals(type)) {
+ return "list<" + displayIcebergType(icebergType.get("element")) + ">";
+ }
+ if ("map".equals(type)) {
+ return "map<" + displayIcebergType(icebergType.get("key")) + ", "
+ + displayIcebergType(icebergType.get("value")) + ">";
+ }
+ if ("fixed".equals(type)) {
+ return "fixed[" + icebergType.path("length").asInt() + "]";
+ }
+ if ("decimal".equals(type)) {
+ return "decimal(" + icebergType.path("precision").asInt() + ", "
+ + icebergType.path("scale").asInt() + ")";
+ }
+ return type == null ? "unknown" : type;
+ }
+
+ private static String baseType(JsonNode icebergType) {
+ if (icebergType == null || icebergType.isNull()) {
+ return null;
+ }
+ String type;
+ if (icebergType.isTextual()) {
+ type = icebergType.asText().trim().toLowerCase(Locale.ROOT);
+ } else if (icebergType.isObject() && icebergType.has("type")) {
+ type = icebergType.get("type").asText().trim().toLowerCase(Locale.ROOT);
+ } else {
+ return null;
+ }
+
+ if (type.startsWith("decimal(")) {
+ return "decimal";
+ }
+ if (type.startsWith("fixed[")) {
+ return "fixed";
+ }
+ if (type.startsWith("geometry(")) {
+ return "geometry";
+ }
+ if (type.startsWith("geography(")) {
+ return "geography";
+ }
+ return type;
+ }
+
+ static void warn(String fieldName, String message) {
+ System.err.println("Warning: field '" + fieldName + "': " + message);
+ }
+}
diff --git a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiModelParser.java b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiModelParser.java
index 8f9201e..593452b 100644
--- a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiModelParser.java
+++ b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiModelParser.java
@@ -134,17 +134,7 @@ public class OsiModelParser {
ds.setFields(fields);
}
- List<Map<String, Object>> extList = (List<Map<String, Object>>) map.get("custom_extensions");
- if (extList != null) {
- List<CustomExtension> extensions = new ArrayList<>();
- for (Map<String, Object> extMap : extList) {
- CustomExtension ext = new CustomExtension();
- ext.setVendorName((String) extMap.get("vendor_name"));
- ext.setData((String) extMap.get("data"));
- extensions.add(ext);
- }
- ds.setCustomExtensions(extensions);
- }
+ ds.setCustomExtensions(parseCustomExtensions(map));
return ds;
}
@@ -153,6 +143,7 @@ public class OsiModelParser {
private Field parseField(Map<String, Object> map) {
Field field = new Field();
field.setName((String) map.get("name"));
+ field.setDatatype((String) map.get("datatype"));
field.setDescription((String) map.get("description"));
// Dimension
@@ -164,9 +155,26 @@ public class OsiModelParser {
// Expressions
field.setExpressions(parseDialectExpressions(map));
+ field.setCustomExtensions(parseCustomExtensions(map));
return field;
}
+ @SuppressWarnings("unchecked")
+ private List<CustomExtension> parseCustomExtensions(Map<String, Object> map) {
+ List<CustomExtension> extensions = new ArrayList<>();
+ List<Map<String, Object>> extList = (List<Map<String, Object>>) map.get("custom_extensions");
+ if (extList == null) {
+ return extensions;
+ }
+ for (Map<String, Object> extMap : extList) {
+ CustomExtension ext = new CustomExtension();
+ ext.setVendorName((String) extMap.get("vendor_name"));
+ ext.setData((String) extMap.get("data"));
+ extensions.add(ext);
+ }
+ return extensions;
+ }
+
@SuppressWarnings("unchecked")
private Relationship parseRelationship(Map<String, Object> map) {
Relationship rel = new Relationship();
diff --git a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiYamlGenerator.java b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiYamlGenerator.java
index 2533b26..d1c6933 100644
--- a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiYamlGenerator.java
+++ b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiYamlGenerator.java
@@ -22,10 +22,7 @@ package org.apache.ossie.converter.polaris;
import org.apache.ossie.converter.polaris.model.OsiModel;
import org.apache.ossie.converter.polaris.model.OsiModel.*;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
import java.util.List;
-import java.util.Map;
/**
* Generates Ossie YAML from an {@link OsiModel}.
@@ -107,17 +104,17 @@ public class OsiYamlGenerator {
}
if (!ds.getCustomExtensions().isEmpty()) {
- sb.append(" custom_extensions:\n");
- for (CustomExtension ext : ds.getCustomExtensions()) {
- sb.append(" - vendor_name: ").append(ext.getVendorName()).append("\n");
- sb.append(" data: '").append(ext.getData()).append("'\n");
- }
+ generateCustomExtensions(sb, ds.getCustomExtensions(), " ");
}
}
private void generateField(StringBuilder sb, Field field) {
sb.append(" - name: ").append(field.getName()).append("\n");
+ if (field.getDatatype() != null) {
+ sb.append(" datatype: ").append(field.getDatatype()).append("\n");
+ }
+
if (!field.getExpressions().isEmpty()) {
sb.append(" expression:\n");
sb.append(" dialects:\n");
@@ -142,6 +139,20 @@ public class OsiYamlGenerator {
if (field.getDescription() != null) {
sb.append(" description: \"").append(escapeYaml(field.getDescription())).append("\"\n");
}
+
+ if (!field.getCustomExtensions().isEmpty()) {
+ generateCustomExtensions(sb, field.getCustomExtensions(), " ");
+ }
+ }
+
+ private void generateCustomExtensions(
+ StringBuilder sb, List<CustomExtension> extensions, String indent) {
+ sb.append(indent).append("custom_extensions:\n");
+ for (CustomExtension ext : extensions) {
+ sb.append(indent).append(" - vendor_name: ").append(ext.getVendorName()).append("\n");
+ sb.append(indent).append(" data: '")
+ .append(ext.getData().replace("'", "''")).append("'\n");
+ }
}
private void generateRelationship(StringBuilder sb, Relationship rel) {
diff --git a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisExporter.java b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisExporter.java
index a864cbd..4909563 100644
--- a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisExporter.java
+++ b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisExporter.java
@@ -19,18 +19,22 @@
package org.apache.ossie.converter.polaris;
+import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.TextNode;
import org.apache.ossie.converter.polaris.model.OsiModel;
import org.apache.ossie.converter.polaris.model.OsiModel.*;
import java.io.IOException;
-import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicInteger;
/**
* Exports an Ossie semantic model to an Apache Polaris catalog.
@@ -113,13 +117,14 @@ public class PolarisExporter {
ArrayNode fields = schema.putArray("fields");
List<String> pk = dataset.getPrimaryKey();
+ AtomicInteger nextNestedId = new AtomicInteger(dataset.getFields().size() + 1);
int fieldId = 1;
for (Field osiField : dataset.getFields()) {
ObjectNode field = objectMapper.createObjectNode();
field.put("id", fieldId);
field.put("name", osiField.getName());
- field.put("type", inferIcebergType(osiField));
+ field.set("type", inferIcebergType(osiField, nextNestedId));
field.put("required", pk != null && pk.contains(osiField.getName()));
if (osiField.getDescription() != null) {
field.put("doc", osiField.getDescription());
@@ -143,13 +148,52 @@ public class PolarisExporter {
}
/**
- * Infer an Iceberg type from an Ossie field.
+ * Resolve an Iceberg type from an Ossie field.
* <p>
- * Since Ossie fields are expression-based and don't carry explicit type information,
- * we use heuristics based on field name, description, and dimension metadata.
+ * Exact Polaris extension data wins, followed by the portable Ossie datatype.
+ * Legacy description, temporal-role, and name heuristics remain as fallbacks for
+ * models authored before datatype support.
*/
- private String inferIcebergType(Field field) {
- // Check description for Iceberg type hint (from round-trip)
+ private JsonNode inferIcebergType(Field field, AtomicInteger nextNestedId) {
+ JsonNode exactType = IcebergTypeMapper.exactIcebergType(field, objectMapper);
+ if (exactType != null) {
+ String extensionDatatype = IcebergTypeMapper.toOssieDatatype(exactType);
+ if (field.getDatatype() != null
+ && !Objects.equals(field.getDatatype(), extensionDatatype)) {
+ IcebergTypeMapper.warn(
+ field.getName(),
+ "datatype '" + field.getDatatype() + "' conflicts with exact Iceberg type '"
+ + IcebergTypeMapper.displayIcebergType(exactType)
+ + "'; preserving the POLARIS extension value");
+ }
+ return IcebergTypeMapper.prepareExactTypeForExport(exactType, nextNestedId);
+ }
+
+ JsonNode portableType = IcebergTypeMapper.toDefaultIcebergType(field.getDatatype());
+ if (portableType != null) {
+ if ("Decimal".equals(field.getDatatype())) {
+ IcebergTypeMapper.warn(
+ field.getName(),
+ "Ossie datatype 'Decimal' has no precision or scale; using decimal(18, 2)");
+ }
+ return portableType;
+ }
+
+ if (field.getDatatype() != null) {
+ if ("Opaque".equals(field.getDatatype())) {
+ IcebergTypeMapper.warn(
+ field.getName(),
+ "Ossie datatype 'Opaque' has no exact Iceberg type in a POLARIS extension; "
+ + "using legacy inference");
+ } else {
+ IcebergTypeMapper.warn(
+ field.getName(),
+ "unrecognized Ossie datatype '" + field.getDatatype()
+ + "'; using legacy inference");
+ }
+ }
+
+ // Check the legacy description type hint produced by older importer versions.
if (field.getDescription() != null && field.getDescription().startsWith("Iceberg type: ")) {
String typeHint = field.getDescription().substring("Iceberg type: ".length());
// Strip optional/required suffix
@@ -157,16 +201,19 @@ public class PolarisExporter {
if (parenIdx > 0) {
typeHint = typeHint.substring(0, parenIdx);
}
- return typeHint;
+ return TextNode.valueOf(typeHint);
}
- // Time dimension -> timestamp
+ // A time role is only a fallback; it never overrides an explicit datatype.
if (field.isTime()) {
- return "timestamptz";
+ return TextNode.valueOf("timestamptz");
}
- // Name-based heuristics
- String name = field.getName().toLowerCase();
+ return TextNode.valueOf(inferIcebergTypeFromName(field.getName()));
+ }
+
+ private String inferIcebergTypeFromName(String fieldName) {
+ String name = fieldName.toLowerCase(Locale.ROOT);
if (name.endsWith("_id") || name.equals("id")) {
return "long";
}
diff --git a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisImporter.java b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisImporter.java
index ef4841c..b00e113 100644
--- a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisImporter.java
+++ b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisImporter.java
@@ -20,6 +20,7 @@
package org.apache.ossie.converter.polaris;
import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.ossie.converter.polaris.model.OsiModel;
import org.apache.ossie.converter.polaris.model.OsiModel.*;
@@ -177,20 +178,33 @@ public class PolarisImporter {
*/
private Field mapColumnToField(JsonNode column) {
String name = column.get("name").asText();
- String icebergType = resolveType(column.get("type"));
+ JsonNode icebergTypeNode = column.get("type");
+ String icebergType = IcebergTypeMapper.displayIcebergType(icebergTypeNode);
+ String datatype = IcebergTypeMapper.toOssieDatatype(icebergTypeNode);
Field field = new Field();
field.setName(name);
+ field.setDatatype(datatype);
// The expression is just the column name (direct mapping)
DialectExpression expr = new DialectExpression("ANSI_SQL", name);
field.setExpressions(Collections.singletonList(expr));
- // Detect time-based dimensions from Iceberg types
- if (isTemporalType(icebergType)) {
+ // Preserve the converter's existing temporal-role classification.
+ if (IcebergTypeMapper.isTemporalDatatype(datatype)) {
field.setTime(true);
}
+ // DataType is intentionally logical and loses physical details such as
+ // integer width, decimal precision/scale, timestamp precision, and nested
+ // structure. Preserve the complete Iceberg type for exact round trips.
+ if (icebergTypeNode != null) {
+ ObjectNode extensionData = client.getObjectMapper().createObjectNode();
+ extensionData.set(IcebergTypeMapper.ICEBERG_TYPE_KEY, icebergTypeNode.deepCopy());
+ field.setCustomExtensions(Collections.singletonList(
+ new CustomExtension(IcebergTypeMapper.POLARIS_VENDOR, extensionData.toString())));
+ }
+
// Add type information as description
field.setDescription("Iceberg type: " + icebergType
+ (isRequired(column) ? " (required)" : " (optional)"));
@@ -198,48 +212,6 @@ public class PolarisImporter {
return field;
}
- /**
- * Resolve an Iceberg type node to a type string.
- * Handles both primitive types (strings) and complex types (struct, list, map).
- */
- private String resolveType(JsonNode typeNode) {
- if (typeNode == null) {
- return "unknown";
- }
- if (typeNode.isTextual()) {
- return typeNode.asText();
- }
- if (typeNode.isObject()) {
- String type = typeNode.has("type") ? typeNode.get("type").asText() : "unknown";
- switch (type) {
- case "struct":
- return "struct";
- case "list":
- String elementType = resolveType(typeNode.path("element"));
- return "list<" + elementType + ">";
- case "map":
- String keyType = resolveType(typeNode.path("key"));
- String valueType = resolveType(typeNode.path("value"));
- return "map<" + keyType + ", " + valueType + ">";
- case "fixed":
- return "fixed[" + typeNode.path("length").asInt() + "]";
- case "decimal":
- return "decimal(" + typeNode.path("precision").asInt()
- + ", " + typeNode.path("scale").asInt() + ")";
- default:
- return type;
- }
- }
- return "unknown";
- }
-
- private boolean isTemporalType(String icebergType) {
- return "timestamp".equals(icebergType)
- || "timestamptz".equals(icebergType)
- || "date".equals(icebergType)
- || "time".equals(icebergType);
- }
-
private boolean isRequired(JsonNode column) {
JsonNode required = column.get("required");
return required != null && required.asBoolean(false);
diff --git a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/model/OsiModel.java b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/model/OsiModel.java
index e36a5e1..bb19888 100644
--- a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/model/OsiModel.java
+++ b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/model/OsiModel.java
@@ -106,13 +106,18 @@ public class OsiModel {
public static class Field {
private String name;
+ private String datatype;
private String description;
private List<DialectExpression> expressions = new ArrayList<>();
private boolean isTime;
+ private List<CustomExtension> customExtensions = new ArrayList<>();
public String getName() { return name; }
public void setName(String name) { this.name = name; }
+ public String getDatatype() { return datatype; }
+ public void setDatatype(String datatype) { this.datatype = datatype; }
+
public String getDescription() { return description; }
public void setDescription(String description) { this.description = description; }
@@ -121,6 +126,9 @@ public class OsiModel {
public boolean isTime() { return isTime; }
public void setTime(boolean time) { isTime = time; }
+
+ public List<CustomExtension> getCustomExtensions() { return customExtensions; }
+ public void setCustomExtensions(List<CustomExtension> customExtensions) { this.customExtensions = customExtensions; }
}
public static class DialectExpression {
diff --git a/converters/polaris/src/test/java/org/apache/ossie/converter/polaris/IcebergTypeMapperTest.java b/converters/polaris/src/test/java/org/apache/ossie/converter/polaris/IcebergTypeMapperTest.java
new file mode 100644
index 0000000..a9d2bd1
--- /dev/null
+++ b/converters/polaris/src/test/java/org/apache/ossie/converter/polaris/IcebergTypeMapperTest.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ossie.converter.polaris;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.TextNode;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+class IcebergTypeMapperTest {
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ @ParameterizedTest
+ @MethodSource("icebergToOssieMappings")
+ void testIcebergToOssieMappings(JsonNode icebergType, String datatype) {
+ assertEquals(datatype, IcebergTypeMapper.toOssieDatatype(icebergType));
+ }
+
+ static Stream<Arguments> icebergToOssieMappings() throws Exception {
+ return Stream.of(
+ Arguments.of(TextNode.valueOf("boolean"), "Boolean"),
+ Arguments.of(TextNode.valueOf("int"), "Integer"),
+ Arguments.of(TextNode.valueOf("long"), "Integer"),
+ Arguments.of(TextNode.valueOf("float"), "Float"),
+ Arguments.of(TextNode.valueOf("double"), "Float"),
+ Arguments.of(TextNode.valueOf("decimal(18,2)"), "Decimal"),
+ Arguments.of(TextNode.valueOf("date"), "Date"),
+ Arguments.of(TextNode.valueOf("time"), "Time"),
+ Arguments.of(TextNode.valueOf("timestamp"), "DateTime"),
+ Arguments.of(TextNode.valueOf("timestamp_ns"), "DateTime"),
+ Arguments.of(TextNode.valueOf("timestamptz"), "DateTimeTz"),
+ Arguments.of(TextNode.valueOf("timestamptz_ns"), "DateTimeTz"),
+ Arguments.of(TextNode.valueOf("string"), "String"),
+ Arguments.of(TextNode.valueOf("uuid"), "Opaque"),
+ Arguments.of(TextNode.valueOf("fixed[16]"), "Opaque"),
+ Arguments.of(TextNode.valueOf("binary"), "Opaque"),
+ Arguments.of(TextNode.valueOf("variant"), "Opaque"),
+ Arguments.of(TextNode.valueOf("geometry(srid:4326)"), "Opaque"),
+ Arguments.of(TextNode.valueOf("geography(srid:4326, spherical)"), "Opaque"),
+ Arguments.of(OBJECT_MAPPER.readTree(
+ "{\"type\":\"list\",\"element-id\":2,\"element\":\"string\","
+ + "\"element-required\":false}"), "Opaque"),
+ Arguments.of(OBJECT_MAPPER.readTree(
+ "{\"type\":\"map\",\"key-id\":2,\"key\":\"string\","
+ + "\"value-id\":3,\"value\":\"long\",\"value-required\":false}"),
+ "Opaque"),
+ Arguments.of(OBJECT_MAPPER.readTree("{\"type\":\"struct\",\"fields\":[]}"), "Opaque"));
+ }
+
+ @Test
+ void testIcebergUnknownOmitsDatatype() {
+ assertNull(IcebergTypeMapper.toOssieDatatype(TextNode.valueOf("unknown")));
+ assertNull(IcebergTypeMapper.toOssieDatatype(null));
+ }
+
+ @ParameterizedTest
+ @MethodSource("ossieToIcebergMappings")
+ void testOssieToIcebergMappings(String datatype, String icebergType) {
+ assertEquals(icebergType, IcebergTypeMapper.toDefaultIcebergType(datatype).asText());
+ }
+
+ static Stream<Arguments> ossieToIcebergMappings() {
+ return Stream.of(
+ Arguments.of("String", "string"),
+ Arguments.of("Integer", "long"),
+ Arguments.of("Decimal", "decimal(18, 2)"),
+ Arguments.of("Float", "double"),
+ Arguments.of("Boolean", "boolean"),
+ Arguments.of("Date", "date"),
+ Arguments.of("Time", "time"),
+ Arguments.of("DateTime", "timestamp"),
+ Arguments.of("DateTimeTz", "timestamptz"));
+ }
+
+ @Test
+ void testOpaqueAndUnknownHaveNoDefaultIcebergType() {
+ assertNull(IcebergTypeMapper.toDefaultIcebergType("Opaque"));
+ assertNull(IcebergTypeMapper.toDefaultIcebergType("Geography"));
+ assertNull(IcebergTypeMapper.toDefaultIcebergType(null));
+ }
+
+ @Test
+ void testNestedTypeIdsAreReassignedWithoutMutatingSource() throws Exception {
+ JsonNode source = OBJECT_MAPPER.readTree(
+ "{\"type\":\"struct\",\"fields\":["
+ + "{\"id\":8,\"name\":\"tags\",\"required\":false,"
+ + "\"type\":{\"type\":\"list\",\"element-id\":9,"
+ + "\"element\":\"string\",\"element-required\":false}},"
+ + "{\"id\":10,\"name\":\"properties\",\"required\":false,"
+ + "\"type\":{\"type\":\"map\",\"key-id\":11,\"key\":\"string\","
+ + "\"value-id\":12,\"value\":\"long\",\"value-required\":false}}]}"
+ );
+
+ JsonNode exported = IcebergTypeMapper.prepareExactTypeForExport(source, new AtomicInteger(4));
+
+ assertEquals(4, exported.path("fields").get(0).path("id").asInt());
+ assertEquals(5, exported.path("fields").get(0).path("type").path("element-id").asInt());
+ assertEquals(6, exported.path("fields").get(1).path("id").asInt());
+ assertEquals(7, exported.path("fields").get(1).path("type").path("key-id").asInt());
+ assertEquals(8, exported.path("fields").get(1).path("type").path("value-id").asInt());
+ assertEquals(8, source.path("fields").get(0).path("id").asInt());
+ assertEquals(9, source.path("fields").get(0).path("type").path("element-id").asInt());
+ }
+
+ @Test
+ void testLegacyObjectDecimalIsCanonicalizedForExport() throws Exception {
+ JsonNode source = OBJECT_MAPPER.readTree(
+ "{\"type\":\"decimal\",\"precision\":20,\"scale\":4}");
+
+ JsonNode exported = IcebergTypeMapper.prepareExactTypeForExport(source, new AtomicInteger(1));
+
+ assertEquals("decimal(20, 4)", exported.asText());
+ }
+}
diff --git a/converters/polaris/src/test/java/org/apache/ossie/converter/polaris/OsiPolarisConverterTest.java b/converters/polaris/src/test/java/org/apache/ossie/converter/polaris/OsiPolarisConverterTest.java
index 33eba4b..456bdb2 100644
--- a/converters/polaris/src/test/java/org/apache/ossie/converter/polaris/OsiPolarisConverterTest.java
+++ b/converters/polaris/src/test/java/org/apache/ossie/converter/polaris/OsiPolarisConverterTest.java
@@ -26,8 +26,9 @@ import org.apache.ossie.converter.polaris.model.OsiModel;
import org.apache.ossie.converter.polaris.model.OsiModel.*;
import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -48,17 +49,23 @@ class OsiPolarisConverterTest {
+ " description: Order fact table\n"
+ " fields:\n"
+ " - name: order_id\n"
+ + " datatype: Integer\n"
+ " expression:\n"
+ " dialects:\n"
+ " - dialect: ANSI_SQL\n"
+ " expression: order_id\n"
+ " - name: total_amount\n"
+ + " datatype: Decimal\n"
+ " expression:\n"
+ " dialects:\n"
+ " - dialect: ANSI_SQL\n"
+ " expression: \"quantity * unit_price\"\n"
+ " description: Computed total\n"
+ + " custom_extensions:\n"
+ + " - vendor_name: POLARIS\n"
+ + " data: '{\"iceberg_type\":\"decimal(18,2)\"}'\n"
+ " - name: order_date\n"
+ + " datatype: Date\n"
+ " expression:\n"
+ " dialects:\n"
+ " - dialect: ANSI_SQL\n"
@@ -70,11 +77,13 @@ class OsiPolarisConverterTest {
+ " primary_key: [customer_id]\n"
+ " fields:\n"
+ " - name: customer_id\n"
+ + " datatype: String\n"
+ " expression:\n"
+ " dialects:\n"
+ " - dialect: ANSI_SQL\n"
+ " expression: customer_id\n"
+ " - name: full_name\n"
+ + " datatype: String\n"
+ " expression:\n"
+ " dialects:\n"
+ " - dialect: ANSI_SQL\n"
@@ -125,6 +134,7 @@ class OsiPolarisConverterTest {
Field computed = orders.getFields().get(1);
assertEquals("total_amount", computed.getName());
+ assertEquals("Decimal", computed.getDatatype());
assertEquals("quantity * unit_price", computed.getExpressions().get(0).getExpression());
}
@@ -137,6 +147,7 @@ class OsiPolarisConverterTest {
Dataset orders = model.getSemanticModels().get(0).getDatasets().get(0);
Field orderDate = orders.getFields().get(2);
assertEquals("order_date", orderDate.getName());
+ assertEquals("Date", orderDate.getDatatype());
assertTrue(orderDate.isTime());
}
@@ -172,6 +183,8 @@ class OsiPolarisConverterTest {
assertTrue(yaml.contains("source: catalog.ns.orders"));
assertTrue(yaml.contains("primary_key: [order_id]"));
assertTrue(yaml.contains("name: total_amount"));
+ assertTrue(yaml.contains("datatype: Decimal"));
+ assertTrue(yaml.contains("datatype: Date"));
assertTrue(yaml.contains("is_time: true"));
assertTrue(yaml.contains("name: orders_to_customer"));
assertTrue(yaml.contains("from_columns: [customer_id]"));
@@ -224,11 +237,12 @@ class OsiPolarisConverterTest {
JsonNode amountField = fields.get(1);
assertEquals("total_amount", amountField.get("name").asText());
assertFalse(amountField.get("required").asBoolean());
+ assertEquals("decimal(18,2)", amountField.get("type").asText());
- // order_date should be timestamptz (time dimension)
+ // Explicit datatype determines the physical type independently of its time role.
JsonNode dateField = fields.get(2);
assertEquals("order_date", dateField.get("name").asText());
- assertEquals("timestamptz", dateField.get("type").asText());
+ assertEquals("date", dateField.get("type").asText());
// Verify identifier-field-ids for primary key
JsonNode identifierFieldIds = schema.get("identifier-field-ids");
@@ -259,8 +273,9 @@ class OsiPolarisConverterTest {
+ " {\"id\": 1, \"name\": \"id\", \"type\": \"long\", \"required\": true},\n"
+ " {\"id\": 2, \"name\": \"name\", \"type\": \"string\", \"required\": false},\n"
+ " {\"id\": 3, \"name\": \"created_at\", \"type\": \"timestamptz\", \"required\": false},\n"
- + " {\"id\": 4, \"name\": \"amount\", \"type\": {\"type\": \"decimal\", \"precision\": 18, \"scale\": 2}, \"required\": false},\n"
- + " {\"id\": 5, \"name\": \"tags\", \"type\": {\"type\": \"list\", \"element-id\": 6, \"element\": \"string\", \"element-required\": false}, \"required\": false}\n"
+ + " {\"id\": 4, \"name\": \"amount\", \"type\": \"decimal(18,2)\", \"required\": false},\n"
+ + " {\"id\": 5, \"name\": \"event_nanos\", \"type\": \"timestamp_ns\", \"required\": false},\n"
+ + " {\"id\": 6, \"name\": \"tags\", \"type\": {\"type\": \"list\", \"element-id\": 99, \"element\": \"string\", \"element-required\": false}, \"required\": false}\n"
+ " ],\n"
+ " \"identifier-field-ids\": [1]\n"
+ " }],\n"
@@ -273,44 +288,29 @@ class OsiPolarisConverterTest {
ObjectMapper mapper = new ObjectMapper();
JsonNode tableMetadata = mapper.readTree(tableMetadataJson);
- // Use reflection-free approach: create importer and test via YAML round-trip
- PolarisClient client = new PolarisClient("http://localhost:8181", "test_catalog");
+ PolarisClient client = new FakePolarisClient(tableMetadata);
PolarisImporter importer = new PolarisImporter(client);
-
- // We test the mapping logic by creating a model and verifying YAML output
- OsiModel model = new OsiModel();
- model.setVersion("0.2.0.dev0");
-
- SemanticModel sm = new SemanticModel();
- sm.setName("test_ns");
- sm.setDescription("Test namespace");
-
- // Manually build what the importer would produce
- Dataset ds = new Dataset();
- ds.setName("test_table");
- ds.setSource("test_catalog.test_ns.test_table");
- ds.setPrimaryKey(Collections.singletonList("id"));
-
- // Map fields from the metadata
- JsonNode schema = tableMetadata.get("metadata").get("schemas").get(0);
- JsonNode fields = schema.get("fields");
-
- List<Field> osiFields = new java.util.ArrayList<>();
- for (JsonNode col : fields) {
- Field f = new Field();
- f.setName(col.get("name").asText());
- f.setExpressions(Collections.singletonList(
- new DialectExpression("ANSI_SQL", col.get("name").asText())));
-
- String type = col.get("type").isTextual() ? col.get("type").asText() : col.get("type").get("type").asText();
- if ("timestamptz".equals(type) || "timestamp".equals(type) || "date".equals(type)) {
- f.setTime(true);
- }
- osiFields.add(f);
- }
- ds.setFields(osiFields);
- sm.setDatasets(Collections.singletonList(ds));
- model.setSemanticModels(Collections.singletonList(sm));
+ OsiModel model = importer.importCatalog();
+
+ Dataset ds = model.getSemanticModels().get(0).getDatasets().get(0);
+ assertEquals("test_table", ds.getName());
+ assertEquals("test_catalog.test_ns.test_table", ds.getSource());
+ assertEquals(Collections.singletonList("id"), ds.getPrimaryKey());
+ assertEquals(6, ds.getFields().size());
+
+ assertEquals("Integer", ds.getFields().get(0).getDatatype());
+ assertEquals("String", ds.getFields().get(1).getDatatype());
+ assertEquals("DateTimeTz", ds.getFields().get(2).getDatatype());
+ assertTrue(ds.getFields().get(2).isTime());
+ assertEquals("Decimal", ds.getFields().get(3).getDatatype());
+ assertEquals("DateTime", ds.getFields().get(4).getDatatype());
+ assertTrue(ds.getFields().get(4).isTime());
+ assertEquals("Opaque", ds.getFields().get(5).getDatatype());
+
+ JsonNode tagsExtension = mapper.readTree(
+ ds.getFields().get(5).getCustomExtensions().get(0).getData());
+ assertEquals("list", tagsExtension.path("iceberg_type").path("type").asText());
+ assertEquals(99, tagsExtension.path("iceberg_type").path("element-id").asInt());
// Generate YAML and verify
OsiYamlGenerator generator = new OsiYamlGenerator();
@@ -325,6 +325,86 @@ class OsiPolarisConverterTest {
assertTrue(yaml.contains("is_time: true"));
assertTrue(yaml.contains("name: amount"));
assertTrue(yaml.contains("name: tags"));
+ assertTrue(yaml.contains("datatype: Integer"));
+ assertTrue(yaml.contains("datatype: DateTimeTz"));
+ assertTrue(yaml.contains("datatype: DateTime"));
+ assertTrue(yaml.contains("datatype: Decimal"));
+ assertTrue(yaml.contains("datatype: Opaque"));
+ assertTrue(yaml.contains("vendor_name: POLARIS"));
+
+ // Reparse the generated Ossie YAML and export it again. Exact physical
+ // distinctions survive, while nested IDs are regenerated for the new schema.
+ OsiModel reparsed = new OsiModelParser().parse(
+ new ByteArrayInputStream(yaml.getBytes(StandardCharsets.UTF_8)));
+ Dataset reparsedDataset = reparsed.getSemanticModels().get(0).getDatasets().get(0);
+ JsonNode exportedSchema = mapper.readTree(
+ new PolarisExporter(client).buildCreateTableRequest(reparsedDataset)).path("schema");
+ JsonNode exportedFields = exportedSchema.path("fields");
+ assertEquals("long", exportedFields.get(0).path("type").asText());
+ assertEquals("timestamptz", exportedFields.get(2).path("type").asText());
+ assertEquals("decimal(18,2)", exportedFields.get(3).path("type").asText());
+ assertEquals("timestamp_ns", exportedFields.get(4).path("type").asText());
+ assertEquals("list", exportedFields.get(5).path("type").path("type").asText());
+ assertEquals(7, exportedFields.get(5).path("type").path("element-id").asInt());
+ }
+
+ @Test
+ void testDatatypePrecedenceAndLegacyFallbacks() throws Exception {
+ Dataset ds = new Dataset();
+ ds.setName("precedence_test");
+ ds.setSource("cat.ns.precedence_test");
+
+ Field typedStringWithTimeRole = makeField("typed_string", true);
+ typedStringWithTimeRole.setDatatype("String");
+
+ Field typedLocalTimestamp = makeField("local_timestamp", false);
+ typedLocalTimestamp.setDatatype("DateTime");
+
+ Field legacyUuid = makeField("legacy_value", false);
+ legacyUuid.setDescription("Iceberg type: uuid (optional)");
+
+ Field exactConflict = makeField("exact_conflict", false);
+ exactConflict.setDatatype("String");
+ exactConflict.setCustomExtensions(Collections.singletonList(
+ new CustomExtension("POLARIS", "{\"iceberg_type\":\"int\"}")));
+
+ Field opaqueWithoutExtension = makeField("opaque_id", false);
+ opaqueWithoutExtension.setDatatype("Opaque");
+
+ Field decimalWithoutExtension = makeField("amount", false);
+ decimalWithoutExtension.setDatatype("Decimal");
+
+ ds.setFields(java.util.Arrays.asList(
+ typedStringWithTimeRole,
+ typedLocalTimestamp,
+ legacyUuid,
+ exactConflict,
+ opaqueWithoutExtension,
+ decimalWithoutExtension));
+
+ PolarisClient client = new PolarisClient("http://localhost:8181", "cat");
+ ByteArrayOutputStream warningBytes = new ByteArrayOutputStream();
+ PrintStream originalError = System.err;
+ String requestJson;
+ try {
+ System.setErr(new PrintStream(warningBytes, true, StandardCharsets.UTF_8));
+ requestJson = new PolarisExporter(client).buildCreateTableRequest(ds);
+ } finally {
+ System.setErr(originalError);
+ }
+ JsonNode fields = new ObjectMapper().readTree(requestJson).path("schema").path("fields");
+
+ assertEquals("string", fields.get(0).path("type").asText());
+ assertEquals("timestamp", fields.get(1).path("type").asText());
+ assertEquals("uuid", fields.get(2).path("type").asText());
+ assertEquals("int", fields.get(3).path("type").asText());
+ assertEquals("long", fields.get(4).path("type").asText());
+ assertEquals("decimal(18, 2)", fields.get(5).path("type").asText());
+
+ String warnings = warningBytes.toString(StandardCharsets.UTF_8);
+ assertTrue(warnings.contains("conflicts with exact Iceberg type 'int'"));
+ assertTrue(warnings.contains("datatype 'Opaque' has no exact Iceberg type"));
+ assertTrue(warnings.contains("datatype 'Decimal' has no precision or scale"));
}
@Test
@@ -393,4 +473,28 @@ class OsiPolarisConverterTest {
f.setTime(isTime);
return f;
}
+
+ private static class FakePolarisClient extends PolarisClient {
+ private final JsonNode tableMetadata;
+
+ FakePolarisClient(JsonNode tableMetadata) {
+ super("http://localhost:8181", "test_catalog");
+ this.tableMetadata = tableMetadata;
+ }
+
+ @Override
+ public List<List<String>> listNamespaces() {
+ return Collections.singletonList(Collections.singletonList("test_ns"));
+ }
+
+ @Override
+ public List<String> listTables(List<String> namespace) {
+ return Collections.singletonList("test_table");
+ }
+
+ @Override
+ public JsonNode loadTable(List<String> namespace, String tableName) {
+ return tableMetadata;
+ }
+ }
}
diff --git a/converters/salesforce/README.md b/converters/salesforce/README.md
index 616c8a5..d49ea2f 100644
--- a/converters/salesforce/README.md
+++ b/converters/salesforce/README.md
@@ -21,7 +21,9 @@
A two-way converter between [Ossie semantic models](../../core-spec/spec.md) and [Salesforce Semantic Model](https://developer.salesforce.com/docs/data/semantic-layer/guide/salesforce-semantic-model-schema.html).
-This converter provides lossless, bidirectional conversion between Ossie YAML format and Salesforce Semantic Model JSON format.
+This converter supports conversion in both directions between Ossie YAML and
+Salesforce Semantic Model JSON. Unmapped Salesforce properties are preserved in
+`custom_extensions`; see the mapping reference for direction-specific limits.
## Requirements
@@ -129,7 +131,7 @@ osiToSf.convert(Paths.get("input/model.yaml"), Paths.get("output/"));
- **Schema-validated** - Input is validated against JSON Schema before processing
- **Lossless conversion** - Unmapped properties are preserved in `custom_extensions`
-- **Bidirectional** - Full bi-directional conversion without data loss
+- **Bidirectional** - Supports both directions, with direction-specific limits documented below
- **Supports Ossie Specification v0.2.0.dev0**
## Mapping Reference
@@ -144,6 +146,7 @@ osiToSf.convert(Paths.get("input/model.yaml"), Paths.get("output/"));
| `semanticDataObjects[].dataObjectName` | `datasets[].source` |
| `semanticDimensions[]` + `semanticMeasurements[]` | `fields[]` |
| `dataObjectFieldName` | `expression.dialects[].expression` |
+| Field `dataType` | Field `datatype` |
| `semanticRelationships[]` | `relationships[]` |
| `criteria[]` | `from_columns` + `to_columns` |
| `semanticCalculatedMeasurements[]` | `metrics[]` |
@@ -159,21 +162,68 @@ osiToSf.convert(Paths.get("input/model.yaml"), Paths.get("output/"));
| `datasets[]` | `semanticDataObjects[]` |
| `datasets[].name` | `semanticDataObjects[].apiName` |
| `datasets[].source` | `semanticDataObjects[].dataObjectName` |
-| `fields[]` | Split into `semanticDimensions[]` and `semanticMeasurements[]` based on `expression` analysis |
+| Direct `fields[]` | Split into `semanticDimensions[]` and `semanticMeasurements[]` based on `dimension` presence |
+| Calculated Tableau fields | `semanticCalculatedDimensions[]` through the existing expression-analysis path |
| `expression.dialects[].expression` | `dataObjectFieldName` |
+| Field `datatype` | Field `dataType` when a safe mapping exists |
| `relationships[]` | `semanticRelationships[]` |
| `from_columns` + `to_columns` | `criteria[]` |
-| `metrics[]` | `semanticCalculatedMeasurements[]` |
+| `metrics[]` | Not currently exported |
| `ai_context` | `businessPreferences` |
| `custom_extensions` (vendor: `SALESFORCE`) | Restored properties |
-### Type Detection (Export)
-
-Fields are automatically classified as dimensions or measurements based on expression analysis:
-
-- **Measurements** — expressions containing SQL aggregation functions (`SUM`, `COUNT`, `AVG`, etc.)
-- **Dimensions** — all other fields
-- **Time dimensions** — Date/DateTime types set `dimension.is_time: true`
+### Data Types
+
+Salesforce imports map field and calculated-measurement types to Ossie's portable
+logical `datatype` vocabulary:
+
+| Salesforce `dataType` | Ossie `datatype` |
+|-----------------------|------------------|
+| `Text`, `Email`, `PhoneNumber`, `Url` | `String` |
+| `Number`, `Currency`, `Percentage` | `Decimal` |
+| `Boolean` | `Boolean` |
+| `Date` | `Date` |
+| `DateTime` | `DateTimeTz` |
+| `Geo` or another known vendor type | `Opaque` |
+
+`Number` remains `Decimal` even when `decimalPlace` is zero because
+`decimalPlace` is display metadata, not an integral-value constraint. Missing
+Salesforce types remain unspecified. Exact Salesforce types are also retained in
+the `SALESFORCE` custom extension so distinctions such as `Email` versus `Text`
+and `Currency` versus `Number` round-trip losslessly.
+
+Ossie field export uses these portable defaults when no exact Salesforce extension
+type exists:
+
+| Ossie `datatype` | Salesforce `dataType` |
+|------------------|-----------------------|
+| `String` | `Text` |
+| `Integer`, `Decimal`, `Float` | `Number` |
+| `Boolean` | `Boolean` |
+| `Date` | `Date` |
+| `DateTime`, `DateTimeTz` | `DateTime` |
+| `Time`, `Opaque` | Omitted with a warning unless an exact extension type exists |
+
+Salesforce has one `DateTime` type, so exporting timezone-free Ossie `DateTime`
+loses its distinction from `DateTimeTz`; the converter logs a warning because a
+subsequent Salesforce import interprets that value as `DateTimeTz`.
+
+An exact Salesforce extension value takes precedence over the portable mapping.
+If it conflicts with `datatype`, the converter preserves the exact Salesforce
+value and logs a warning.
+
+### Field Role and Time Dimensions
+
+`datatype` does not determine whether an Ossie field is a dimension or a fact.
+For direct fields, the presence of the `dimension` object determines whether the
+field is exported to `semanticDimensions` or `semanticMeasurements`. A calculated
+Tableau expression follows the converter's existing calculated-dimension path.
+
+On import, Salesforce `Date` and `DateTime` dimensions set `dimension.is_time` to
+`true`; other dimension types set it to `false`. On export, `dimension.is_time`
+does not invent or override a scalar type. This preserves Ossie's separation of
+logical data type from temporal role, including integer year and string month
+dimensions.
### Relationship Handling
diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java
index 19d68cc..787b7af 100644
--- a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java
+++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java
@@ -65,6 +65,7 @@ public final class ConverterConstants {
public static final String LABEL = "label";
public static final String DESCRIPTION = "description";
public static final String DATA_TYPE = "dataType";
+ public static final String OSI_DATATYPE = "datatype";
public static final String AI_CONTEXT = "ai_context";
public static final String BUSINESS_PREFERENCES = "businessPreferences";
@@ -108,10 +109,6 @@ public final class ConverterConstants {
public static final String FIELD_TYPE_SEMANTIC_FIELD = "SemanticField";
public static final String FIELD_TYPE_FORMULA = "Formula";
- // Data type values
- public static final String DATA_TYPE_DATE = "Date";
- public static final String DATA_TYPE_DATE_TIME = "DateTime";
-
// Default values settings
public static final String CARDINALITY = "cardinality";
public static final String IS_ENABLED = "isEnabled";
diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/FieldMappingHandler.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/FieldMappingHandler.java
index 1d14834..161146c 100644
--- a/converters/salesforce/src/main/java/org/apache/ossie/converter/FieldMappingHandler.java
+++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/FieldMappingHandler.java
@@ -183,7 +183,7 @@ public class FieldMappingHandler implements PipelineStep {
// Add dimension property with is_time based on dataType
Map<String, Object> dimensionProp = new LinkedHashMap<>();
String dataType = getString(sfDimension, DATA_TYPE);
- if (DATA_TYPE_DATE.equals(dataType) || DATA_TYPE_DATE_TIME.equals(dataType)) {
+ if (SalesforceDataTypeMapper.isTemporalSalesforceType(dataType)) {
dimensionProp.put(IS_TIME, true);
} else {
dimensionProp.put(IS_TIME, false);
@@ -238,6 +238,11 @@ public class FieldMappingHandler implements PipelineStep {
if (description != null) {
osiField.put(DESCRIPTION, description);
}
+
+ String datatype = SalesforceDataTypeMapper.toOssie(getString(sfField, DATA_TYPE));
+ if (datatype != null) {
+ osiField.put(OSI_DATATYPE, datatype);
+ }
}
/**
@@ -307,6 +312,8 @@ public class FieldMappingHandler implements PipelineStep {
customExtensionHandler.restoreSalesforceCustomExtension(calcDim, osiField);
+ applyOsiDatatype(calcDim, osiField);
+
applyFieldDefaults(calcDim);
// Add to semantic calculated dimensions array
@@ -320,6 +327,8 @@ public class FieldMappingHandler implements PipelineStep {
customExtensionHandler.restoreSalesforceCustomExtension(sfField, osiField);
+ applyOsiDatatype(sfField, osiField);
+
applyFieldDefaults(sfField);
RoutingResult result =
@@ -539,6 +548,54 @@ public class FieldMappingHandler implements PipelineStep {
sfField.putIfAbsent(DISPLAY_CATEGORY, DISPLAY_CATEGORY_CONTINUOUS);
}
+ /**
+ * Applies the portable Ossie datatype when no exact Salesforce dataType was
+ * restored from custom_extensions. Exact extension data wins to preserve
+ * Salesforce-specific distinctions such as Email and Currency.
+ */
+ private void applyOsiDatatype(Map<String, Object> sfField, Map<String, Object> osiField) {
+ String osiDatatype = getString(osiField, OSI_DATATYPE);
+ String exactSalesforceDataType = getString(sfField, DATA_TYPE);
+ String mappedSalesforceDataType = SalesforceDataTypeMapper.toSalesforce(osiDatatype);
+ String selectedSalesforceDataType = exactSalesforceDataType != null
+ ? exactSalesforceDataType
+ : mappedSalesforceDataType;
+
+ if (SalesforceDataTypeMapper.isTimezoneLossyMapping(
+ osiDatatype, selectedSalesforceDataType)) {
+ logger.warn(
+ "Field '{}' has Ossie datatype 'DateTime'; Salesforce dataType 'DateTime' "
+ + "cannot preserve the timezone-free distinction and re-imports as 'DateTimeTz'",
+ getString(osiField, NAME));
+ }
+
+ if (exactSalesforceDataType != null) {
+ if (osiDatatype != null
+ && !SalesforceDataTypeMapper.areCompatible(osiDatatype, exactSalesforceDataType)) {
+ logger.warn(
+ "Field '{}' has Ossie datatype '{}' that conflicts with exact Salesforce dataType '{}'; "
+ + "preserving the Salesforce extension value",
+ getString(osiField, NAME),
+ osiDatatype,
+ exactSalesforceDataType);
+ }
+ return;
+ }
+
+ if (osiDatatype == null) {
+ return;
+ }
+
+ if (mappedSalesforceDataType == null) {
+ logger.warn(
+ "Field '{}' has Ossie datatype '{}' with no safe Salesforce mapping; omitting dataType",
+ getString(osiField, NAME),
+ osiDatatype);
+ return;
+ }
+ sfField.put(DATA_TYPE, mappedSalesforceDataType);
+ }
+
/**
* Processes model-level semanticCalculatedDimensions and converts them to dataset fields
* if all their dependencies point to the same data object.
@@ -649,7 +706,7 @@ public class FieldMappingHandler implements PipelineStep {
// Add dimension property with is_time based on dataType
Map<String, Object> dimensionProp = new LinkedHashMap<>();
String dataType = getString(calcDim, DATA_TYPE);
- if (DATA_TYPE_DATE.equals(dataType) || DATA_TYPE_DATE_TIME.equals(dataType)) {
+ if (SalesforceDataTypeMapper.isTemporalSalesforceType(dataType)) {
dimensionProp.put(IS_TIME, true);
} else {
dimensionProp.put(IS_TIME, false);
diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java
index c54d46a..a84e3f4 100644
--- a/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java
+++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/MetricMappingHandler.java
@@ -132,6 +132,11 @@ public class MetricMappingHandler implements PipelineStep {
// Wrap in Ossie dialect structure
osiMetric.put(EXPRESSION, wrapExpression(expressionValue));
}
+
+ String datatype = SalesforceDataTypeMapper.toOssie(getString(sfMetric, DATA_TYPE));
+ if (datatype != null) {
+ osiMetric.put(OSI_DATATYPE, datatype);
+ }
}
}
diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/SalesforceDataTypeMapper.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/SalesforceDataTypeMapper.java
new file mode 100644
index 0000000..78a7258
--- /dev/null
+++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/SalesforceDataTypeMapper.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ossie.converter;
+
+import java.util.Map;
+import java.util.Objects;
+
+/** Maps Salesforce semantic data types to and from Ossie's portable logical types. */
+final class SalesforceDataTypeMapper {
+
+ private static final Map<String, String> SALESFORCE_TO_OSSIE = Map.ofEntries(
+ Map.entry("Text", "String"),
+ Map.entry("Email", "String"),
+ Map.entry("PhoneNumber", "String"),
+ Map.entry("Url", "String"),
+ Map.entry("Number", "Decimal"),
+ Map.entry("Currency", "Decimal"),
+ Map.entry("Percentage", "Decimal"),
+ Map.entry("Boolean", "Boolean"),
+ Map.entry("Date", "Date"),
+ Map.entry("DateTime", "DateTimeTz"),
+ Map.entry("Geo", "Opaque"));
+
+ private static final Map<String, String> OSSIE_TO_SALESFORCE = Map.ofEntries(
+ Map.entry("String", "Text"),
+ Map.entry("Integer", "Number"),
+ Map.entry("Decimal", "Number"),
+ Map.entry("Float", "Number"),
+ Map.entry("Boolean", "Boolean"),
+ Map.entry("Date", "Date"),
+ Map.entry("DateTime", "DateTime"),
+ Map.entry("DateTimeTz", "DateTime"));
+
+ private SalesforceDataTypeMapper() {}
+
+ /**
+ * Maps a Salesforce type to an Ossie logical type. Unknown non-empty Salesforce
+ * types are known vendor types outside the portable vocabulary and become Opaque.
+ */
+ static String toOssie(String salesforceDataType) {
+ if (salesforceDataType == null || salesforceDataType.isBlank()) {
+ return null;
+ }
+ return SALESFORCE_TO_OSSIE.getOrDefault(salesforceDataType, "Opaque");
+ }
+
+ /** Maps an Ossie logical type to Salesforce, or returns null when no safe mapping exists. */
+ static String toSalesforce(String osiDatatype) {
+ if (osiDatatype == null || osiDatatype.isBlank()) {
+ return null;
+ }
+ return OSSIE_TO_SALESFORCE.get(osiDatatype);
+ }
+
+ static boolean isTemporalSalesforceType(String salesforceDataType) {
+ return "Date".equals(salesforceDataType) || "DateTime".equals(salesforceDataType);
+ }
+
+ /** Returns whether this mapping loses Ossie's timezone-free DateTime distinction. */
+ static boolean isTimezoneLossyMapping(String osiDatatype, String salesforceDataType) {
+ return "DateTime".equals(osiDatatype) && "DateTime".equals(salesforceDataType);
+ }
+
+ /**
+ * Returns whether an exact Salesforce extension type and a portable Ossie type
+ * describe compatible values. This treats lossy portable collapses such as
+ * Email/String and Currency/Decimal as compatible.
+ */
+ static boolean areCompatible(String osiDatatype, String salesforceDataType) {
+ if (osiDatatype == null || salesforceDataType == null) {
+ return true;
+ }
+ return Objects.equals(toSalesforce(osiDatatype), salesforceDataType)
+ || Objects.equals(toOssie(salesforceDataType), osiDatatype);
+ }
+}
diff --git a/converters/salesforce/src/test/java/org/apache/ossie/OsiToSalesforceConverterTest.java b/converters/salesforce/src/test/java/org/apache/ossie/OsiToSalesforceConverterTest.java
index 94fa6b2..c298df6 100644
--- a/converters/salesforce/src/test/java/org/apache/ossie/OsiToSalesforceConverterTest.java
+++ b/converters/salesforce/src/test/java/org/apache/ossie/OsiToSalesforceConverterTest.java
@@ -266,6 +266,24 @@ class OsiToSalesforceConverterTest {
assertNotNull(customerIdDim);
assertEquals("Text", customerIdDim.get("dataType"));
assertEquals("Discrete", customerIdDim.get("displayCategory"));
+
+ Map<String, Object> emailDim = customerDimensions.stream()
+ .filter(d -> "email".equals(d.get("apiName")))
+ .findFirst()
+ .orElse(null);
+ assertNotNull(emailDim);
+ assertEquals("Email", emailDim.get("dataType"),
+ "Exact Salesforce extension type should win over portable String");
+
+ List<Map<String, Object>> customerMeasurements =
+ (List<Map<String, Object>>) customersDataset.get("semanticMeasurements");
+ Map<String, Object> lifetimeValue = customerMeasurements.stream()
+ .filter(m -> "lifetime_value".equals(m.get("apiName")))
+ .findFirst()
+ .orElse(null);
+ assertNotNull(lifetimeValue);
+ assertEquals("Currency", lifetimeValue.get("dataType"),
+ "Exact Salesforce extension type should win over portable Decimal");
}
@Test
diff --git a/converters/salesforce/src/test/java/org/apache/ossie/SalesforceToOsiConverterTest.java b/converters/salesforce/src/test/java/org/apache/ossie/SalesforceToOsiConverterTest.java
index 1926843..2cdae4b 100644
--- a/converters/salesforce/src/test/java/org/apache/ossie/SalesforceToOsiConverterTest.java
+++ b/converters/salesforce/src/test/java/org/apache/ossie/SalesforceToOsiConverterTest.java
@@ -155,9 +155,24 @@ class SalesforceToOsiConverterTest {
.orElse(null);
assertNotNull(customerIdField);
assertEquals("Customer ID", customerIdField.get("label"));
+ assertEquals("String", customerIdField.get("datatype"));
assertNotNull(customerIdField.get("dimension"));
assertNotNull(customerIdField.get("expression"));
+ Map<String, Object> emailField = fields.stream()
+ .filter(f -> "email".equals(f.get("name")))
+ .findFirst()
+ .orElse(null);
+ assertNotNull(emailField);
+ assertEquals("String", emailField.get("datatype"));
+
+ Map<String, Object> purchasesField = fields.stream()
+ .filter(f -> "total_purchases".equals(f.get("name")))
+ .findFirst()
+ .orElse(null);
+ assertNotNull(purchasesField);
+ assertEquals("Decimal", purchasesField.get("datatype"));
+
// Verify expression dialect structure
Map<String, Object> expression = (Map<String, Object>) customerIdField.get("expression");
List<Map<String, Object>> dialects = (List<Map<String, Object>>) expression.get("dialects");
@@ -187,6 +202,11 @@ class SalesforceToOsiConverterTest {
boolean hasOrderYear = orderFields.stream()
.anyMatch(f -> "order_year".equals(f.get("name")));
assertTrue(hasOrderYear, "order_year should be converted to a field");
+ Map<String, Object> orderYear = orderFields.stream()
+ .filter(f -> "order_year".equals(f.get("name")))
+ .findFirst()
+ .orElseThrow();
+ assertEquals("Decimal", orderYear.get("datatype"));
// customer_order_key should remain in custom_extensions (multiple dependencies)
List<Map<String, Object>> customExtensions = (List<Map<String, Object>>) model.get("custom_extensions");
@@ -260,6 +280,7 @@ class SalesforceToOsiConverterTest {
// Verify first metric
Map<String, Object> metric1 = metrics.get(0);
assertEquals("total_revenue", metric1.get("name"));
+ assertEquals("Decimal", metric1.get("datatype"));
assertNotNull(metric1.get("expression"));
Map<String, Object> expression = (Map<String, Object>) metric1.get("expression");
@@ -308,6 +329,7 @@ class SalesforceToOsiConverterTest {
.findFirst()
.orElse(null);
assertNotNull(orderDateField);
+ assertEquals("Date", orderDateField.get("datatype"));
Map<String, Object> dimension = (Map<String, Object>) orderDateField.get("dimension");
assertNotNull(dimension);
diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/SalesforceDataTypeMapperTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/SalesforceDataTypeMapperTest.java
new file mode 100644
index 0000000..c49c3d4
--- /dev/null
+++ b/converters/salesforce/src/test/java/org/apache/ossie/converter/SalesforceDataTypeMapperTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ossie.converter;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.util.stream.Stream;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+class SalesforceDataTypeMapperTest {
+
+ @ParameterizedTest
+ @MethodSource("salesforceToOssieTypes")
+ void mapsSalesforceTypesToOssie(String salesforceType, String osiDatatype) {
+ assertEquals(osiDatatype, SalesforceDataTypeMapper.toOssie(salesforceType));
+ }
+
+ static Stream<Arguments> salesforceToOssieTypes() {
+ return Stream.of(
+ Arguments.of("Text", "String"),
+ Arguments.of("Email", "String"),
+ Arguments.of("PhoneNumber", "String"),
+ Arguments.of("Url", "String"),
+ Arguments.of("Number", "Decimal"),
+ Arguments.of("Currency", "Decimal"),
+ Arguments.of("Percentage", "Decimal"),
+ Arguments.of("Boolean", "Boolean"),
+ Arguments.of("Date", "Date"),
+ Arguments.of("DateTime", "DateTimeTz"),
+ Arguments.of("Geo", "Opaque"),
+ Arguments.of("Duration", "Opaque"),
+ Arguments.of("FutureVendorType", "Opaque"));
+ }
+
+ @ParameterizedTest
+ @MethodSource("ossieToSalesforceTypes")
+ void mapsOssieTypesToSalesforce(String osiDatatype, String salesforceType) {
+ assertEquals(salesforceType, SalesforceDataTypeMapper.toSalesforce(osiDatatype));
+ }
+
+ static Stream<Arguments> ossieToSalesforceTypes() {
+ return Stream.of(
+ Arguments.of("String", "Text"),
+ Arguments.of("Integer", "Number"),
+ Arguments.of("Decimal", "Number"),
+ Arguments.of("Float", "Number"),
+ Arguments.of("Boolean", "Boolean"),
+ Arguments.of("Date", "Date"),
+ Arguments.of("DateTime", "DateTime"),
+ Arguments.of("DateTimeTz", "DateTime"));
+ }
+
+ @Test
+ void omitsMissingAndUnrepresentableTypes() {
+ assertNull(SalesforceDataTypeMapper.toOssie(null));
+ assertNull(SalesforceDataTypeMapper.toOssie(" "));
+ assertNull(SalesforceDataTypeMapper.toSalesforce(null));
+ assertNull(SalesforceDataTypeMapper.toSalesforce("Time"));
+ assertNull(SalesforceDataTypeMapper.toSalesforce("Opaque"));
+ assertNull(SalesforceDataTypeMapper.toSalesforce("FutureOssieType"));
+ }
+
+ @Test
+ void recognizesTemporalSalesforceTypes() {
+ assertTrue(SalesforceDataTypeMapper.isTemporalSalesforceType("Date"));
+ assertTrue(SalesforceDataTypeMapper.isTemporalSalesforceType("DateTime"));
+ assertFalse(SalesforceDataTypeMapper.isTemporalSalesforceType("Text"));
+ assertFalse(SalesforceDataTypeMapper.isTemporalSalesforceType(null));
+ }
+
+ @Test
+ void identifiesTimezoneLossyRoundTrips() {
+ String localDateTimeTarget = SalesforceDataTypeMapper.toSalesforce("DateTime");
+ String instantTarget = SalesforceDataTypeMapper.toSalesforce("DateTimeTz");
+
+ assertEquals("DateTimeTz", SalesforceDataTypeMapper.toOssie(localDateTimeTarget));
+ assertEquals("DateTimeTz", SalesforceDataTypeMapper.toOssie(instantTarget));
+ assertTrue(SalesforceDataTypeMapper.isTimezoneLossyMapping("DateTime", localDateTimeTarget));
+ assertFalse(SalesforceDataTypeMapper.isTimezoneLossyMapping("DateTimeTz", instantTarget));
+ assertFalse(SalesforceDataTypeMapper.isTimezoneLossyMapping("Date", "Date"));
+ }
+
+ @Test
+ void treatsPortableCollapsesAsCompatibleWithExactSalesforceTypes() {
+ assertTrue(SalesforceDataTypeMapper.areCompatible("String", "Email"));
+ assertTrue(SalesforceDataTypeMapper.areCompatible("Decimal", "Currency"));
+ assertTrue(SalesforceDataTypeMapper.areCompatible("Integer", "Number"));
+ assertTrue(SalesforceDataTypeMapper.areCompatible("DateTimeTz", "DateTime"));
+ assertTrue(SalesforceDataTypeMapper.areCompatible("Opaque", "Geo"));
+ assertFalse(SalesforceDataTypeMapper.areCompatible("String", "Number"));
+ }
+}
diff --git a/converters/salesforce/src/test/resources/examples/osiToSalesforce.yaml b/converters/salesforce/src/test/resources/examples/osiToSalesforce.yaml
index edb1859..101588f 100644
--- a/converters/salesforce/src/test/resources/examples/osiToSalesforce.yaml
+++ b/converters/salesforce/src/test/resources/examples/osiToSalesforce.yaml
@@ -39,6 +39,7 @@ semantic_model:
}
fields:
- name: customer_id
+ datatype: String
label: Customer ID
description: Unique customer identifier
dimension:
@@ -55,6 +56,7 @@ semantic_model:
"displayCategory" : "Discrete"
}
- name: email
+ datatype: String
label: Email
description: Customer email address
dimension:
@@ -70,6 +72,7 @@ semantic_model:
"dataType" : "Email"
}
- name: total_purchases
+ datatype: Decimal
label: Total Purchases
description: Lifetime purchase count
expression:
@@ -85,6 +88,7 @@ semantic_model:
"displayCategory" : "Continuous"
}
- name: lifetime_value
+ datatype: Decimal
label: Lifetime Value
description: Total customer value
expression:
@@ -99,6 +103,7 @@ semantic_model:
"aggregationType" : "Sum"
}
- name: customer_email_domain
+ datatype: String
label: Customer Email Domain
description: Customer Email Domain
dimension:
@@ -126,6 +131,7 @@ semantic_model:
}
fields:
- name: order_id
+ datatype: String
label: Order ID
description: Unique order identifier
dimension:
@@ -141,6 +147,7 @@ semantic_model:
"dataType" : "Text"
}
- name: customer_id
+ datatype: String
label: Customer ID
description: Foreign key to Customers
dimension:
@@ -159,6 +166,7 @@ semantic_model:
"sortOrder" : "Ascending"
}
- name: product_id
+ datatype: String
label: Product ID
description: Foreign key to Products
dimension:
@@ -177,6 +185,7 @@ semantic_model:
"sortOrder" : "Ascending"
}
- name: order_date
+ datatype: Date
label: Order Date
description: Date when order was placed
dimension:
@@ -195,6 +204,7 @@ semantic_model:
"sortOrder" : "Descending"
}
- name: amount
+ datatype: Decimal
label: Amount
description: Order amount in currency
expression:
@@ -209,6 +219,7 @@ semantic_model:
"aggregationType" : "Sum"
}
- name: quantity
+ datatype: Decimal
label: Quantity
description: Number of items ordered
expression:
@@ -223,6 +234,7 @@ semantic_model:
"aggregationType" : "Sum"
}
- name: order_year
+ datatype: Decimal
label: Order Year
description: Order Year
dimension:
@@ -250,6 +262,7 @@ semantic_model:
}
fields:
- name: product_id
+ datatype: String
label: Product ID
description: Unique product identifier
dimension:
@@ -268,6 +281,7 @@ semantic_model:
"sortOrder" : "Ascending"
}
- name: product_name
+ datatype: String
label: Product Name
description: Product display name
dimension:
@@ -286,6 +300,7 @@ semantic_model:
"sortOrder" : "Ascending"
}
- name: unit_price
+ datatype: Decimal
label: Unit Price
description: Price per unit
expression:
@@ -299,6 +314,7 @@ semantic_model:
"dataType" : "Currency"
}
- name: stock_level
+ datatype: Decimal
label: Stock Level
description: Current inventory count
expression:
@@ -373,12 +389,14 @@ semantic_model:
metrics:
- description: Sum of all order amounts
name: total_revenue
+ datatype: Decimal
expression:
dialects:
- dialect: ANSI_SQL
expression: SUM([Orders].[amount])
- description: Average amount per order
name: avg_order_value
+ datatype: Decimal
expression:
dialects:
- dialect: ANSI_SQL
diff --git a/converters/snowflake/README.md b/converters/snowflake/README.md
index 0026e0a..656954c 100644
--- a/converters/snowflake/README.md
+++ b/converters/snowflake/README.md
@@ -35,6 +35,26 @@ uv sync
uv run ossie-snowflake -i input.yaml -o output.yaml
```
+## Data Type Mapping
+
+The exporter maps the optional logical `datatype` on Ossie fields to Snowflake
+`data_type` as follows:
+
+| Ossie | Snowflake |
+|---|---|
+| `String` | `VARCHAR` |
+| `Integer` | `NUMBER(38,0)` |
+| `Decimal` | `NUMBER` |
+| `Float` | `FLOAT` |
+| `Boolean` | `BOOLEAN` |
+| `Date` | `DATE` |
+| `Time` | `TIME` |
+| `DateTime` | `TIMESTAMP_NTZ` |
+| `DateTimeTz` | `TIMESTAMP_TZ` |
+
+An omitted datatype remains unspecified. `Opaque` has no portable Snowflake
+mapping, so the exporter omits `data_type` and emits a warning.
+
## Tests
```bash
@@ -44,3 +64,6 @@ uv run pytest
## Limitations
Some Ossie concepts (e.g., `ai_context` on relationships) do not have a native counterpart in the Snowflake semantic model. These are dropped during conversion and the converter will emit warnings so you know what was left behind.
+
+Snowflake metric result types are inferred from their expressions, so Ossie
+metric `datatype` values are not emitted as `data_type` properties.
diff --git a/converters/snowflake/src/ossie_snowflake/converter.py b/converters/snowflake/src/ossie_snowflake/converter.py
index 124ef62..6dcd77f 100644
--- a/converters/snowflake/src/ossie_snowflake/converter.py
+++ b/converters/snowflake/src/ossie_snowflake/converter.py
@@ -33,11 +33,50 @@ import yaml
SUPPORTED_VERSION = "0.2.0.dev0"
+_TIME_DATATYPES = frozenset({"Date", "Time", "DateTime", "DateTimeTz"})
+
+_SNOWFLAKE_DATATYPES = {
+ "String": "VARCHAR",
+ "Integer": "NUMBER(38,0)",
+ "Decimal": "NUMBER",
+ "Float": "FLOAT",
+ "Boolean": "BOOLEAN",
+ "Date": "DATE",
+ "Time": "TIME",
+ "DateTime": "TIMESTAMP_NTZ",
+ "DateTimeTz": "TIMESTAMP_TZ",
+}
+
class OsiConversionError(Exception):
"""Raised when an Ossie YAML cannot be converted to Snowflake format."""
+def _convert_datatype(datatype, field_name):
+ """Map an Ossie logical datatype to a Snowflake field data type.
+
+ Missing datatypes remain unspecified. ``Opaque`` and unrecognized values
+ cannot be mapped safely, so they are omitted with a warning.
+ """
+ if datatype is None:
+ return None
+
+ if datatype == "Opaque":
+ warnings.warn(
+ f"Omitting data_type from field '{field_name}': "
+ "Ossie datatype 'Opaque' has no portable Snowflake mapping"
+ )
+ return None
+
+ snowflake_datatype = _SNOWFLAKE_DATATYPES.get(datatype)
+ if snowflake_datatype is None:
+ warnings.warn(
+ f"Omitting data_type from field '{field_name}': "
+ f"unrecognized Ossie datatype '{datatype}'"
+ )
+ return snowflake_datatype
+
+
def convert_osi_to_snowflake(osi_yaml_str):
"""Top-level entry point. Parses Ossie YAML, validates, converts, returns
Snowflake YAML string.
@@ -191,6 +230,11 @@ def _convert_dataset(dataset):
converted = _convert_named_expr(field, "field")
if converted is None:
continue
+ snowflake_datatype = _convert_datatype(
+ field.get("datatype"), field.get("name", "<unnamed>")
+ )
+ if snowflake_datatype is not None:
+ converted["data_type"] = snowflake_datatype
if classification == "time_dimension":
time_dimensions.append(converted)
elif classification == "dimension":
@@ -216,11 +260,31 @@ def _convert_dataset(dataset):
def _classify_field(field):
- """Returns 'dimension', 'time_dimension', or 'fact' based on field structure."""
+ """Classify a field as 'fact', 'dimension', or 'time_dimension'.
+
+ ``datatype`` declares the field's data type; ``dimension.is_time`` is
+ an independent temporal-role marker. Classification rules:
+
+ - A field with no ``dimension`` block is a ``fact`` regardless of
+ ``datatype`` (data type does not imply role).
+ - Explicit ``dimension.is_time`` always wins: ``True`` classifies as
+ ``time_dimension``; ``False`` classifies as ``dimension`` even when
+ ``datatype`` is temporal (author opt-out for e.g. audit timestamps).
+ - When ``dimension.is_time`` is unset, it defaults to ``True`` for
+ temporal ``datatype`` values (``Date``, ``Time``, ``DateTime``,
+ ``DateTimeTz``) and ``False`` otherwise.
+ """
dimension = field.get("dimension")
if dimension is None:
return "fact"
- if isinstance(dimension, dict) and dimension.get("is_time") is True:
+ is_time = dimension.get("is_time") if isinstance(dimension, dict) else None
+ if is_time is True:
+ return "time_dimension"
+ if is_time is False:
+ return "dimension"
+ # is_time is unset; default from datatype
+ datatype = field.get("datatype")
+ if datatype in _TIME_DATATYPES:
return "time_dimension"
return "dimension"
diff --git a/converters/snowflake/tests/example_converted_tpcds_semantic_model.yaml b/converters/snowflake/tests/example_converted_tpcds_semantic_model.yaml
index b2d4aff..7817666 100644
--- a/converters/snowflake/tests/example_converted_tpcds_semantic_model.yaml
+++ b/converters/snowflake/tests/example_converted_tpcds_semantic_model.yaml
@@ -44,24 +44,28 @@ tables:
synonyms:
- sale date
- transaction date
+ data_type: NUMBER(38,0)
- name: ss_item_sk
expr: ss_item_sk
description: Foreign key to item dimension
synonyms:
- product
- item
+ data_type: NUMBER(38,0)
- name: ss_customer_sk
expr: ss_customer_sk
description: Foreign key to customer dimension
synonyms:
- customer
- buyer
+ data_type: NUMBER(38,0)
- name: ss_store_sk
expr: ss_store_sk
description: Foreign key to store dimension
synonyms:
- store
- location
+ data_type: NUMBER(38,0)
facts:
- name: ss_quantity
expr: ss_quantity
@@ -69,24 +73,28 @@ tables:
synonyms:
- units sold
- quantity
+ data_type: NUMBER(38,0)
- name: ss_sales_price
expr: ss_sales_price
description: Sales price per unit
synonyms:
- unit price
- price
+ data_type: NUMBER
- name: ss_ext_sales_price
expr: ss_ext_sales_price
description: Extended sales price (quantity * price)
synonyms:
- total price
- line total
+ data_type: NUMBER
- name: ss_net_profit
expr: ss_net_profit
description: Net profit from the sale
synonyms:
- profit
- margin
+ data_type: NUMBER
- name: date_dim
base_table:
database: TPCDS
@@ -107,6 +115,7 @@ tables:
- name: d_date_sk
expr: d_date_sk
description: Surrogate key for date
+ data_type: NUMBER(38,0)
time_dimensions:
- name: d_date
expr: d_date
@@ -114,11 +123,13 @@ tables:
synonyms:
- date
- calendar date
+ data_type: DATE
- name: d_year
expr: d_year
description: Year
synonyms:
- year
+ data_type: NUMBER(38,0)
- name: d_quarter_name
expr: d_quarter_name
description: Quarter name (e.g., 2024Q1)
@@ -150,30 +161,36 @@ tables:
- name: c_customer_sk
expr: c_customer_sk
description: Surrogate key for customer
+ data_type: NUMBER(38,0)
- name: c_customer_id
expr: c_customer_id
description: Business key for customer
synonyms:
- customer ID
- customer number
+ data_type: VARCHAR
- name: c_first_name
expr: c_first_name
description: Customer first name
+ data_type: VARCHAR
- name: c_last_name
expr: c_last_name
description: Customer last name
+ data_type: VARCHAR
- name: customer_full_name
expr: c_first_name || ' ' || c_last_name
description: Customer full name (computed field)
synonyms:
- full name
- customer name
+ data_type: VARCHAR
- name: c_email_address
expr: c_email_address
description: Customer email address
synonyms:
- email
- contact
+ data_type: VARCHAR
- name: item
base_table:
database: TPCDS
@@ -194,6 +211,7 @@ tables:
- name: i_item_sk
expr: i_item_sk
description: Surrogate key for item
+ data_type: NUMBER(38,0)
- name: i_item_id
expr: i_item_id
description: Business key for item
@@ -201,30 +219,35 @@ tables:
- item ID
- product ID
- SKU
+ data_type: VARCHAR
- name: i_item_desc
expr: i_item_desc
description: Item description
synonyms:
- product description
- item name
+ data_type: VARCHAR
- name: i_brand
expr: i_brand
description: Brand name
synonyms:
- brand
- manufacturer
+ data_type: VARCHAR
- name: i_category
expr: i_category
description: Item category
synonyms:
- product category
- department
+ data_type: VARCHAR
- name: i_current_price
expr: i_current_price
description: Current price of the item
synonyms:
- price
- list price
+ data_type: NUMBER
- name: store
base_table:
database: TPCDS
@@ -245,30 +268,35 @@ tables:
- name: s_store_sk
expr: s_store_sk
description: Surrogate key for store
+ data_type: NUMBER(38,0)
- name: s_store_id
expr: s_store_id
description: Business key for store
synonyms:
- store ID
- store number
+ data_type: VARCHAR
- name: s_store_name
expr: s_store_name
description: Store name
synonyms:
- store name
- location name
+ data_type: VARCHAR
- name: s_city
expr: s_city
description: City where store is located
synonyms:
- city
- location
+ data_type: VARCHAR
- name: s_state
expr: s_state
description: State where store is located
synonyms:
- state
- region
+ data_type: VARCHAR
facts:
- name: s_number_employees
expr: s_number_employees
@@ -276,6 +304,7 @@ tables:
synonyms:
- employee count
- staff size
+ data_type: NUMBER(38,0)
relationships:
- name: store_sales_to_date
left_table: store_sales
diff --git a/converters/snowflake/tests/test_osi_to_snowflake_yaml_converter.py b/converters/snowflake/tests/test_osi_to_snowflake_yaml_converter.py
index c1b5469..738fdf1 100644
--- a/converters/snowflake/tests/test_osi_to_snowflake_yaml_converter.py
+++ b/converters/snowflake/tests/test_osi_to_snowflake_yaml_converter.py
@@ -26,6 +26,7 @@ from ossie_snowflake.converter import (
OsiConversionError,
convert_osi_to_snowflake,
_classify_field,
+ _convert_datatype,
_convert_dataset,
_convert_named_expr,
_convert_relationship,
@@ -75,6 +76,19 @@ def _minimal_model(**overrides):
return base
+def _typed_field(name, datatype, dimension=None):
+ field = {
+ "name": name,
+ "expression": {
+ "dialects": [{"dialect": "ANSI_SQL", "expression": name}]
+ },
+ "datatype": datatype,
+ }
+ if dimension is not None:
+ field["dimension"] = dimension
+ return field
+
+
# ---------------------------------------------------------------------------
# _normalize_identifier
# ---------------------------------------------------------------------------
@@ -178,6 +192,43 @@ class TestExtractSynonyms:
assert result is not original
+# ---------------------------------------------------------------------------
+# _convert_datatype
+# ---------------------------------------------------------------------------
+
+class TestConvertDatatype:
+ @pytest.mark.parametrize(
+ ("osi_datatype", "snowflake_datatype"),
+ [
+ ("String", "VARCHAR"),
+ ("Integer", "NUMBER(38,0)"),
+ ("Decimal", "NUMBER"),
+ ("Float", "FLOAT"),
+ ("Boolean", "BOOLEAN"),
+ ("Date", "DATE"),
+ ("Time", "TIME"),
+ ("DateTime", "TIMESTAMP_NTZ"),
+ ("DateTimeTz", "TIMESTAMP_TZ"),
+ ],
+ )
+ def test_maps_portable_datatype(self, osi_datatype, snowflake_datatype):
+ assert _convert_datatype(osi_datatype, "field") == snowflake_datatype
+
+ def test_missing_datatype_is_omitted_without_warning(self):
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ assert _convert_datatype(None, "field") is None
+ assert len(caught) == 0
+
+ def test_opaque_datatype_is_omitted_with_warning(self):
+ with pytest.warns(UserWarning, match="Opaque"):
+ assert _convert_datatype("Opaque", "payload") is None
+
+ def test_unrecognized_datatype_is_omitted_with_warning(self):
+ with pytest.warns(UserWarning, match="unrecognized.*Geography"):
+ assert _convert_datatype("Geography", "location") is None
+
+
# ---------------------------------------------------------------------------
# _classify_field
# ---------------------------------------------------------------------------
@@ -199,6 +250,25 @@ class TestClassifyField:
def test_dimension_none_is_fact(self):
assert _classify_field({"dimension": None}) == "fact"
+ @pytest.mark.parametrize(
+ ("field", "classification"),
+ [
+ ({"dimension": {}, "datatype": "Date"}, "time_dimension"),
+ ({"dimension": {}, "datatype": "Time"}, "time_dimension"),
+ ({"dimension": {}, "datatype": "DateTime"}, "time_dimension"),
+ ({"dimension": {}, "datatype": "DateTimeTz"}, "time_dimension"),
+ ({"dimension": {}, "datatype": "String"}, "dimension"),
+ ({"dimension": {}, "datatype": "Decimal"}, "dimension"),
+ ({"dimension": {}, "datatype": "Float"}, "dimension"),
+ ({"dimension": {}, "datatype": "Opaque"}, "dimension"),
+ ({"dimension": {"is_time": True}, "datatype": "Integer"}, "time_dimension"),
+ ({"dimension": {"is_time": False}, "datatype": "DateTimeTz"}, "dimension"),
+ ({"datatype": "DateTime"}, "fact"),
+ ],
+ )
+ def test_datatype_and_explicit_time_role(self, field, classification):
+ assert _classify_field(field) == classification
+
# ---------------------------------------------------------------------------
# _extract_expression
@@ -414,6 +484,42 @@ class TestConvertDataset:
assert len(result["facts"]) == 1
assert result["facts"][0]["name"] == "amount"
+ @pytest.mark.parametrize(
+ ("dimension", "datatype", "bucket", "snowflake_datatype"),
+ [
+ ({"is_time": False}, "String", "dimensions", "VARCHAR"),
+ ({}, "Date", "time_dimensions", "DATE"),
+ (None, "Decimal", "facts", "NUMBER"),
+ ({"is_time": False}, "DateTime", "dimensions", "TIMESTAMP_NTZ"),
+ ({"is_time": True}, "Integer", "time_dimensions", "NUMBER(38,0)"),
+ (None, "DateTimeTz", "facts", "TIMESTAMP_TZ"),
+ ],
+ )
+ def test_datatype_emitted_independently_of_field_role(
+ self, dimension, datatype, bucket, snowflake_datatype
+ ):
+ result = _convert_dataset(
+ {
+ "name": "typed_table",
+ "source": "db.schema.table",
+ "fields": [_typed_field("typed_field", datatype, dimension)],
+ }
+ )
+
+ assert result[bucket][0]["data_type"] == snowflake_datatype
+
+ def test_opaque_field_omits_data_type(self):
+ dataset = {
+ "name": "typed_table",
+ "source": "db.schema.table",
+ "fields": [_typed_field("payload", "Opaque")],
+ }
+
+ with pytest.warns(UserWarning, match="Opaque"):
+ result = _convert_dataset(dataset)
+
+ assert "data_type" not in result["facts"][0]
+
def test_missing_name_raises(self):
with pytest.raises(OsiConversionError, match="Missing required 'name'"):
_convert_dataset({"source": "db.s.t"})
@@ -486,12 +592,14 @@ class TestConvertOsiToSnowflake:
]
},
"description": "Total x",
+ "datatype": "Decimal",
}
]
)
result = yaml.safe_load(convert_osi_to_snowflake(_wrap_osi(model)))
assert len(result["metrics"]) == 1
assert result["metrics"][0]["expr"] == "SUM(x)"
+ assert "data_type" not in result["metrics"][0]
def test_invalid_yaml_root_raises(self):
with pytest.raises(OsiConversionError, match="expected a mapping"):
diff --git a/core-spec/osi-schema.json b/core-spec/osi-schema.json
index 163db20..f24e45f 100644
--- a/core-spec/osi-schema.json
+++ b/core-spec/osi-schema.json
@@ -108,13 +108,29 @@
"required": ["dialects"],
"additionalProperties": false
},
+ "DataType": {
+ "type": "string",
+ "enum": [
+ "String",
+ "Integer",
+ "Decimal",
+ "Float",
+ "Boolean",
+ "Date",
+ "Time",
+ "DateTime",
+ "DateTimeTz",
+ "Opaque"
+ ],
+ "description": "Logical data type for fields and metrics, independent of role (e.g. dimension vs fact) and physical representation. `Decimal` is exact base-10 with unspecified precision and scale; `Float` is approximate. `DateTime` has no timezone or offset, while `DateTimeTz` identifies an instant using offset or timezone context but does not guarantee preservation of a named timezone. Omit `datatype` when unknown; use `Opaque` plus `custom_extensions` for a known type outside the [...]
+ },
"Dimension": {
"type": "object",
"description": "Dimension metadata",
"properties": {
"is_time": {
"type": "boolean",
- "description": "Indicates if this is a time-based dimension for temporal filtering"
+ "description": "Temporal-role marker. When true, consumers that distinguish time dimensions (e.g. for time-series analysis or temporal filtering) should treat this field as a time dimension. This is a *role* flag, independent of the field's data type: a field with `is_time: true` may carry any `datatype` (e.g. `Integer` for a year grain, `String` for a month name, as well as temporal data types). When `is_time` is unset, it defaults to `true` if `datatype` is one of `Date`, `Ti [...]
}
},
"additionalProperties": false
@@ -141,6 +157,9 @@
"type": "string",
"description": "Human-readable description"
},
+ "datatype": {
+ "$ref": "#/$defs/DataType"
+ },
"ai_context": {
"$ref": "#/$defs/AIContext"
},
@@ -266,6 +285,9 @@
"type": "string",
"description": "Human-readable description of what the metric measures"
},
+ "datatype": {
+ "$ref": "#/$defs/DataType"
+ },
"ai_context": {
"$ref": "#/$defs/AIContext"
},
diff --git a/core-spec/spec.md b/core-spec/spec.md
index b7c285e..50f7479 100644
--- a/core-spec/spec.md
+++ b/core-spec/spec.md
@@ -59,6 +59,26 @@ Supported SQL and expression language dialects for metrics and field definitions
| `MAQL` | GoodData MAQL (Metric Analysis and Query Language) |
| `BIGQUERY` | Google BigQuery (GoogleSQL) |
+### Data types
+
+`DataType` declares the logical value type of a field or metric independently
+of its role and physical representation. The shared names align with the
+ontology specification's built-in value types; `Time`, `DateTimeTz`, and
+`Opaque` are additional core data types.
+
+| DataType | Description |
+|----------|-------------|
+| `String` | Variable-length Unicode character data; length and collation are unspecified. |
+| `Integer` | Exact integral number; width and signedness are unspecified. |
+| `Decimal` | Exact base-10 number; precision and scale are unspecified. |
+| `Float` | Approximate floating-point number. |
+| `Boolean` | Logical two-valued truth type. |
+| `Date` | Calendar date with no time-of-day component. |
+| `Time` | Time-of-day with no date or timezone. |
+| `DateTime` | Local/civil date and time with no timezone or offset. |
+| `DateTimeTz` | Date and time with sufficient offset or timezone context to identify an instant. Preservation of a named timezone identifier is not guaranteed. |
+| `Opaque` | Known type outside the portable vocabulary; use `custom_extensions` for vendor-specific refinement. Omit `datatype` when the type is unknown or unspecified. |
+
## Semantic Model
The top-level container that represents a complete semantic model, including datasets, relationships, and metrics.
@@ -213,6 +233,7 @@ Fields represent row-level attributes that can be used for grouping, filtering,
| `dimension` | object | No | Dimension metadata (e.g., `is_time` flag) |
| `label` | string | No | Label for categorization |
| `description` | string | No | Human-readable description |
+| `datatype` | string (enum) | No | Logical data type for this field. See [Data types](#data-types). |
| `ai_context` | string/object | No | Additional context for AI tools (e.g., synonyms) |
| `custom_extensions` | array | No | Vendor-specific attributes |
@@ -239,7 +260,7 @@ expression:
| Field | Type | Description |
|-------|------|-------------|
-| `is_time` | boolean | Indicates if this is a time-based dimension for temporal filtering |
+| `is_time` | boolean | Temporal-role marker. When `true`, consumers that distinguish time dimensions (e.g. for time-series analysis or temporal filtering) should treat this field as a time dimension. This is a *role* flag, independent of the field's data type. See [DataType and `is_time`: type vs. role](#datatype-and-is_time-type-vs-role). |
### Examples
@@ -279,6 +300,7 @@ expression:
dialects:
- dialect: ANSI_SQL
expression: order_date
+ datatype: Date
dimension:
is_time: true
description: Date when order was placed
@@ -303,6 +325,33 @@ expression:
description: Normalized email address
```
+### DataType and `is_time`: type vs. role
+
+`datatype` and `dimension.is_time` are independent properties that answer different questions:
+
+- **`datatype`** describes the *data type* of the field (e.g. `Date`, `Integer`, `String`, `DateTimeTz`): what kind of values the field holds.
+- **`dimension.is_time`** is a *temporal-role marker*: whether the field should be treated as a time dimension for time-series analysis or temporal filtering, regardless of its data type.
+
+**Default for `is_time`.** When `is_time` is not set explicitly, it defaults to `true` if `datatype` is one of `Date`, `Time`, `DateTime`, `DateTimeTz`, and `false` otherwise. Explicit `is_time` always wins. Set `is_time: false` on a temporal-typed column (e.g. an audit `created_at` you don't want on the time axis) to opt out of the default.
+
+Common combinations:
+
+| Column example | `datatype` | `is_time` | Effective role | Why |
+|---|---|---|---|---|
+| `d_date` (calendar date) | `Date` | omitted | time dimension | Temporal `datatype`; `is_time` defaults to `true`. |
+| `order_timestamp` | `DateTimeTz` | omitted | time dimension | Same. |
+| `created_at` (audit timestamp) | `DateTime` | `false` | regular dimension | Explicit opt-out of the temporal default. |
+| `d_year` (integer year grain) | `Integer` | `true` | time dimension | Non-temporal `datatype`; `is_time: true` makes the role explicit. |
+| `d_quarter_name` (e.g. `"Q1"`) | `String` | `true` | time dimension | String-valued temporal grain. |
+| `customer_id` | `Integer` | omitted | regular dimension | Non-temporal `datatype`; `is_time` defaults to `false`. |
+
+> **Precedent.** This type/role separation mirrors [Snowflake Semantic Views' YAML authoring form](https://docs.snowflake.com/en/user-guide/views-semantic/semantic-view-yaml-spec), which has a structural `time_dimensions:` collection whose entries can carry any `data_type`. The published example annotates `order_year` with `data_type: NUMBER`. LookML supports a similar split via its [`dimension_group`](https://cloud.google.com/looker/docs/reference/param-field-dimension-group), whose `da [...]
+
+**Consumer guidance.**
+
+- For *data-type* questions (casting, serialization, downstream type inference): prefer `datatype` when present. If only `is_time: true` is set, do not infer a specific scalar type from it.
+- For *role* questions (classifying time dimensions in a query UI, generating time-series output sections, choosing time-aware aggregations): treat the field as a time dimension when `is_time` resolves to `true`, whether explicitly set or defaulted from a temporal `datatype`.
+
---
## Metrics
@@ -316,6 +365,7 @@ Quantitative measures defined on business data, representing key calculations li
| `name` | string | Yes | Unique identifier for the metric |
| `expression` | object | Yes | Expression definition with dialect support |
| `description` | string | No | Human-readable description of what the metric measures |
+| `datatype` | string (enum) | No | Logical data type for this metric. See [Data types](#data-types). |
| `ai_context` | string/object | No | Additional context for AI tools (e.g., synonyms) |
| `custom_extensions` | array | No | Vendor-specific attributes |
@@ -337,9 +387,11 @@ expression:
```yaml
- name: total_revenue
expression:
- - dialect: ANSI_SQL
- expression: SUM(orders.amount)
+ dialects:
+ - dialect: ANSI_SQL
+ expression: SUM(orders.amount)
description: Total revenue across all orders
+ datatype: Decimal
ai_context:
synonyms:
- "total sales"
@@ -351,9 +403,11 @@ expression:
```yaml
- name: avg_orders
expression:
- - dialect: ANSI_SQL
- expression: SUM(orders.amount) / COUNT(DISTINCT customers.id)
+ dialects:
+ - dialect: ANSI_SQL
+ expression: SUM(orders.amount) / COUNT(DISTINCT customers.id)
description: Average orders
+ datatype: Decimal
ai_context:
synonyms:
- "Order Average by customer"
@@ -477,6 +531,7 @@ semantic_model:
dialects:
- dialect: ANSI_SQL
expression: order_date
+ datatype: Date
dimension:
is_time: true
description: Order date
diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml
index 2c9c73e..b0a921a 100644
--- a/core-spec/spec.yaml
+++ b/core-spec/spec.yaml
@@ -38,7 +38,20 @@ dialects:
- "MAQL" # GoodData MAQL (Multi-Dimensional Analytical Query Language)
- "BIGQUERY" # Google BigQuery GoogleSQL
-
+# Supported logical data types for fields and metrics
+# TODO: Generate this list from the authoritative DataType enum in
+# osi-schema.json ($defs.DataType.enum); until then, keep them in sync.
+datatypes:
+ - "String" # Variable-length Unicode character data
+ - "Integer" # Exact integral number
+ - "Decimal" # Exact base-10 number
+ - "Float" # Approximate floating-point number
+ - "Boolean" # Logical two-valued truth type
+ - "Date" # Calendar date without time of day
+ - "Time" # Time of day without a date or timezone
+ - "DateTime" # Date and time without a timezone or offset
+ - "DateTimeTz" # Instant identified using offset or timezone context
+ - "Opaque" # Known type outside the portable vocabulary
# Vendor name for custom extensions (free-form string)
# Examples: "COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA", "WISDOM"
@@ -184,8 +197,17 @@ fields:
# Optional: Dimension metadata
# Indicates this field can be used as a dimension for grouping/filtering
dimension:
- # Optional: Indicates if this is a time-based dimension
- # Used for time-series analysis and temporal filtering
+ # Optional: Temporal-role marker
+ # When true, consumers should treat this field as a time dimension
+ # for time-series analysis and temporal filtering. This is a *role*
+ # flag, independent of the field's data type. A field with
+ # is_time: true may carry any datatype (e.g. Integer for a year
+ # grain, String for a month name, Date/DateTime for a date column).
+ #
+ # Default: when unset, is_time defaults to true if datatype is one
+ # of Date, Time, DateTime, DateTimeTz, and false otherwise. Set
+ # is_time: false explicitly to opt a temporal-typed column out of
+ # time-dimension treatment.
is_time: boolean
# Optional: Label for categorization (e.g., "filter")
@@ -194,6 +216,14 @@ fields:
# Optional: Human-readable description of the field
description: string
+ # Optional: Logical data type for this field
+ # Must be one of the values from the 'datatypes' enum above.
+ # Decimal is exact base-10 with unspecified precision and scale;
+ # Float is approximate. Omit datatype when unknown or unspecified.
+ # Use Opaque + custom_extensions for a known type outside the
+ # portable vocabulary.
+ datatype: string
+
# Optional: Additional context for AI tools (e.g., synonyms, business terms)
# Helps LLMs understand the field meaning and generate better queries
ai_context: string
@@ -223,6 +253,14 @@ metrics:
# Should explain what the metric measures and how it's used
description: string
+ # Optional: Logical data type for this metric
+ # Must be one of the values from the 'datatypes' enum above.
+ # Decimal is exact base-10 with unspecified precision and scale;
+ # Float is approximate. Omit datatype when unknown or unspecified.
+ # Use Opaque + custom_extensions for a known type outside the
+ # portable vocabulary.
+ datatype: string
+
# Optional: Additional context for AI tools (e.g., synonyms, business context)
# Helps LLMs understand the metric meaning and suggest it appropriately
ai_context: string
diff --git a/docs/index.md b/docs/index.md
index 63febfe..738c572 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -316,7 +316,7 @@ A practical guide for organizations looking to adopt Ossie.
|------|------------|
| **Semantic Model** | A structured description of business data that defines datasets, fields, relationships, and metrics. It provides a shared vocabulary for interpreting data across tools and teams. |
| **Dataset** | A logical representation of a business entity, typically corresponding to a fact table or dimension table in a data warehouse. |
-| **Field** | A row-level attribute within a dataset, used for grouping, filtering, or as part of metric expressions. Fields can be simple column references or computed expressions. |
+| **Field** | A row-level attribute within a dataset, used for grouping, filtering, or as part of metric expressions. Fields can be simple column references or computed expressions. A field's logical data type is declared by the optional top-level `datatype` field (one of `String`, `Integer`, `Decimal`, `Float`, `Boolean`, `Date`, `Time`, `DateTime`, `DateTimeTz`, or `Opaque`). |
| **Dimension** | A categorical attribute used to slice and filter data (e.g., region, product category, date). In Ossie, dimensions are represented as fields with optional metadata such as `is_time`. |
| **Metric** | A quantitative measure computed by aggregating data across one or more datasets (e.g., total revenue, average order value). Metrics are defined at the semantic model level. |
| **Relationship** | A foreign key connection between two datasets, defining how they can be joined. Relationships are always many-to-one (from the referencing dataset to the referenced dataset). |
diff --git a/examples/tpcds_semantic_model.yaml b/examples/tpcds_semantic_model.yaml
index c20f288..a04b168 100644
--- a/examples/tpcds_semantic_model.yaml
+++ b/examples/tpcds_semantic_model.yaml
@@ -50,6 +50,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: ss_sold_date_sk
description: Foreign key to date dimension
+ datatype: Integer
dimension:
is_time: false
ai_context:
@@ -63,6 +64,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: ss_item_sk
description: Foreign key to item dimension
+ datatype: Integer
dimension:
is_time: false
ai_context:
@@ -76,6 +78,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: ss_customer_sk
description: Foreign key to customer dimension
+ datatype: Integer
dimension:
is_time: false
ai_context:
@@ -89,6 +92,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: ss_store_sk
description: Foreign key to store dimension
+ datatype: Integer
dimension:
is_time: false
ai_context:
@@ -102,6 +106,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: ss_quantity
description: Quantity of items sold
+ datatype: Integer
ai_context:
synonyms:
- "units sold"
@@ -113,6 +118,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: ss_sales_price
description: Sales price per unit
+ datatype: Decimal
ai_context:
synonyms:
- "unit price"
@@ -124,6 +130,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: ss_ext_sales_price
description: Extended sales price (quantity * price)
+ datatype: Decimal
ai_context:
synonyms:
- "total price"
@@ -135,6 +142,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: ss_net_profit
description: Net profit from the sale
+ datatype: Decimal
ai_context:
synonyms:
- "profit"
@@ -160,6 +168,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: d_date_sk
description: Surrogate key for date
+ datatype: Integer
dimension:
is_time: false
@@ -169,8 +178,8 @@ semantic_model:
- dialect: ANSI_SQL
expression: d_date
description: Actual date value
- dimension:
- is_time: true
+ datatype: Date
+ dimension: {}
ai_context:
synonyms:
- "date"
@@ -182,12 +191,15 @@ semantic_model:
- dialect: ANSI_SQL
expression: d_year
description: Year
+ datatype: Integer
dimension:
is_time: true
ai_context:
synonyms:
- "year"
+ # Declares temporal role via is_time without a datatype annotation.
+ # Both datatype and is_time are independently optional.
- name: d_quarter_name
expression:
dialects:
@@ -201,6 +213,8 @@ semantic_model:
- "quarter"
- "fiscal quarter"
+ # Declares temporal role via is_time without a datatype annotation.
+ # Both datatype and is_time are independently optional.
- name: d_month_name
expression:
dialects:
@@ -233,6 +247,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: c_customer_sk
description: Surrogate key for customer
+ datatype: Integer
dimension:
is_time: false
@@ -242,6 +257,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: c_customer_id
description: Business key for customer
+ datatype: String
dimension:
is_time: false
ai_context:
@@ -255,6 +271,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: c_first_name
description: Customer first name
+ datatype: String
dimension:
is_time: false
@@ -264,6 +281,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: c_last_name
description: Customer last name
+ datatype: String
dimension:
is_time: false
@@ -273,6 +291,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: c_first_name || ' ' || c_last_name
description: Customer full name (computed field)
+ datatype: String
dimension:
is_time: false
ai_context:
@@ -286,6 +305,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: c_email_address
description: Customer email address
+ datatype: String
dimension:
is_time: false
ai_context:
@@ -313,6 +333,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: i_item_sk
description: Surrogate key for item
+ datatype: Integer
dimension:
is_time: false
@@ -322,6 +343,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: i_item_id
description: Business key for item
+ datatype: String
dimension:
is_time: false
ai_context:
@@ -336,6 +358,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: i_item_desc
description: Item description
+ datatype: String
dimension:
is_time: false
ai_context:
@@ -349,6 +372,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: i_brand
description: Brand name
+ datatype: String
dimension:
is_time: false
ai_context:
@@ -362,6 +386,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: i_category
description: Item category
+ datatype: String
dimension:
is_time: false
ai_context:
@@ -375,6 +400,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: i_current_price
description: Current price of the item
+ datatype: Decimal
dimension:
is_time: false
ai_context:
@@ -402,6 +428,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: s_store_sk
description: Surrogate key for store
+ datatype: Integer
dimension:
is_time: false
@@ -411,6 +438,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: s_store_id
description: Business key for store
+ datatype: String
dimension:
is_time: false
ai_context:
@@ -424,6 +452,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: s_store_name
description: Store name
+ datatype: String
dimension:
is_time: false
ai_context:
@@ -437,6 +466,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: s_city
description: City where store is located
+ datatype: String
dimension:
is_time: false
ai_context:
@@ -450,6 +480,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: s_state
description: State where store is located
+ datatype: String
dimension:
is_time: false
ai_context:
@@ -463,6 +494,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: s_number_employees
description: Number of employees at the store
+ datatype: Integer
ai_context:
synonyms:
- "employee count"
@@ -518,6 +550,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: SUM(store_sales.ss_ext_sales_price)
description: Total sales revenue across all transactions
+ datatype: Decimal
ai_context:
synonyms:
- "total revenue"
@@ -530,6 +563,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: SUM(store_sales.ss_net_profit)
description: Total net profit from store sales
+ datatype: Decimal
ai_context:
synonyms:
- "net profit"
@@ -542,6 +576,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk)
description: Average lifetime sales value per customer
+ datatype: Decimal
ai_context:
synonyms:
- "CLV"
@@ -555,6 +590,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: SUM(store_sales.ss_ext_sales_price)
description: Total sales by brand (requires grouping by item.i_brand)
+ datatype: Decimal
ai_context:
synonyms:
- "brand sales"
@@ -567,6 +603,7 @@ semantic_model:
- dialect: ANSI_SQL
expression: SUM(store_sales.ss_ext_sales_price) / NULLIF(SUM(store.s_number_employees), 0)
description: Sales per employee across stores
+ datatype: Decimal
ai_context:
synonyms:
- "sales per employee"
diff --git a/python/src/ossie/__init__.py b/python/src/ossie/__init__.py
index cb105fa..c7b3360 100644
--- a/python/src/ossie/__init__.py
+++ b/python/src/ossie/__init__.py
@@ -20,6 +20,7 @@ from ossie.models import (
OSIAIContextObject,
OSICustomExtension,
OSIDataset,
+ OSIDataType,
OSIDialect,
OSIDialectExpression,
OSIDimension,
@@ -37,6 +38,7 @@ __all__ = [
"OSIAIContextObject",
"OSICustomExtension",
"OSIDataset",
+ "OSIDataType",
"OSIDialect",
"OSIDialectExpression",
"OSIDimension",
diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py
index 9be74a6..5406a74 100644
--- a/python/src/ossie/models.py
+++ b/python/src/ossie/models.py
@@ -34,6 +34,31 @@ class OSIDialect(str, Enum):
BIGQUERY = "BIGQUERY"
+class OSIDataType(str, Enum):
+ """Portable logical data types for fields and metric results."""
+
+ STRING = "String"
+ INTEGER = "Integer"
+ DECIMAL = "Decimal"
+ FLOAT = "Float"
+ BOOLEAN = "Boolean"
+ DATE = "Date"
+ TIME = "Time"
+ DATE_TIME = "DateTime"
+ DATE_TIME_TZ = "DateTimeTz"
+ OPAQUE = "Opaque"
+
+
+_TEMPORAL_DATA_TYPES = frozenset(
+ {
+ OSIDataType.DATE,
+ OSIDataType.TIME,
+ OSIDataType.DATE_TIME,
+ OSIDataType.DATE_TIME_TZ,
+ }
+)
+
+
class OSIVendor(str, Enum):
"""Well-known vendor names for custom extensions."""
@@ -104,9 +129,23 @@ class OSIField(BaseModel):
dimension: Optional[OSIDimension] = None
label: Optional[str] = None
description: Optional[str] = None
+ datatype: Optional[OSIDataType] = None
ai_context: Optional[OSIAIContext] = None
custom_extensions: Optional[list[OSICustomExtension]] = None
+ def is_time_dimension(self) -> bool:
+ """Return the field's effective temporal-dimension role.
+
+ A field must have dimension metadata to be a dimension. Within that
+ block, an explicit ``is_time`` value takes precedence; otherwise the
+ role defaults from a temporal ``datatype``.
+ """
+ if self.dimension is None:
+ return False
+ if self.dimension.is_time is not None:
+ return self.dimension.is_time
+ return self.datatype in _TEMPORAL_DATA_TYPES
+
class OSIDataset(BaseModel):
"""Logical dataset representing a business entity (fact or dimension table)."""
@@ -145,6 +184,7 @@ class OSIMetric(BaseModel):
name: str
expression: OSIExpression
description: Optional[str] = None
+ datatype: Optional[OSIDataType] = None
ai_context: Optional[OSIAIContext] = None
custom_extensions: Optional[list[OSICustomExtension]] = None
diff --git a/python/tests/test_models.py b/python/tests/test_models.py
new file mode 100644
index 0000000..749a226
--- /dev/null
+++ b/python/tests/test_models.py
@@ -0,0 +1,137 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+from pathlib import Path
+
+import pytest
+import yaml
+from pydantic import ValidationError
+
+from ossie import (
+ OSIDataType,
+ OSIDimension,
+ OSIDocument,
+ OSIExpression,
+ OSIField,
+)
+
+
+def _expression_data(value: str = "value") -> dict:
+ return {"dialects": [{"dialect": "ANSI_SQL", "expression": value}]}
+
+
+def _expression(value: str = "value") -> OSIExpression:
+ return OSIExpression.model_validate(_expression_data(value))
+
+
+def _document() -> dict:
+ return {
+ "version": "0.2.0.dev0",
+ "semantic_model": [
+ {
+ "name": "typed_model",
+ "datasets": [
+ {
+ "name": "events",
+ "source": "catalog.schema.events",
+ "fields": [
+ {
+ "name": "occurred_at",
+ "expression": _expression_data("occurred_at"),
+ "dimension": {},
+ "datatype": "DateTimeTz",
+ }
+ ],
+ }
+ ],
+ "metrics": [
+ {
+ "name": "revenue",
+ "expression": _expression_data("SUM(events.revenue)"),
+ "datatype": "Decimal",
+ }
+ ],
+ }
+ ],
+ }
+
+
+def test_data_type_enum_matches_core_schema() -> None:
+ schema_path = Path(__file__).parents[2] / "core-spec" / "osi-schema.json"
+ schema = json.loads(schema_path.read_text())
+
+ assert [member.value for member in OSIDataType] == schema["$defs"]["DataType"][
+ "enum"
+ ]
+ assert schema["$defs"]["Field"]["properties"]["datatype"] == {
+ "$ref": "#/$defs/DataType"
+ }
+ assert schema["$defs"]["Metric"]["properties"]["datatype"] == {
+ "$ref": "#/$defs/DataType"
+ }
+
+
+def test_field_and_metric_datatypes_survive_serialization() -> None:
+ document = OSIDocument.model_validate(_document())
+
+ field = document.semantic_model[0].datasets[0].fields[0]
+ metric = document.semantic_model[0].metrics[0]
+ assert field.datatype is OSIDataType.DATE_TIME_TZ
+ assert metric.datatype is OSIDataType.DECIMAL
+
+ as_json = json.loads(document.to_osi_json())
+ as_yaml = yaml.safe_load(document.to_osi_yaml())
+ for serialized in (as_json, as_yaml):
+ model = serialized["semantic_model"][0]
+ assert model["datasets"][0]["fields"][0]["datatype"] == "DateTimeTz"
+ assert model["metrics"][0]["datatype"] == "Decimal"
+
+
+def test_invalid_datatype_is_rejected() -> None:
+ document = _document()
+ field = document["semantic_model"][0]["datasets"][0]["fields"][0]
+ field["datatype"] = "timestamp"
+
+ with pytest.raises(ValidationError):
+ OSIDocument.model_validate(document)
+
+
+@pytest.mark.parametrize(
+ ("dimension", "datatype", "expected"),
+ [
+ (None, OSIDataType.DATE, False),
+ (OSIDimension(), OSIDataType.DATE, True),
+ (OSIDimension(is_time=False), OSIDataType.DATE_TIME_TZ, False),
+ (OSIDimension(is_time=True), OSIDataType.STRING, True),
+ (OSIDimension(), OSIDataType.STRING, False),
+ (OSIDimension(), None, False),
+ ],
+)
+def test_effective_time_dimension_role(
+ dimension: OSIDimension | None,
+ datatype: OSIDataType | None,
+ expected: bool,
+) -> None:
+ field = OSIField(
+ name="value",
+ expression=_expression(),
+ dimension=dimension,
+ datatype=datatype,
+ )
+
+ assert field.is_time_dimension() is expected