You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ossie.apache.org by jb...@apache.org on 2026/10/01 14:41:55 UTC
(ossie) branch main updated: Fix SDK version ordering and OrionBelt fallback validation (#429)
This is an automated email from the ASF dual-hosted git repository.
jbonofre pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ossie.git
The following commit(s) were added to refs/heads/main by this push:
new 3cea6fa8 Fix SDK version ordering and OrionBelt fallback validation (#429)
3cea6fa8 is described below
commit 3cea6fa8c186bd515cd1c344150b1d091ae3de08
Author: Yufei Gu <yu...@apache.org>
AuthorDate: Thu Oct 1 07:41:49 2026 -0700
Fix SDK version ordering and OrionBelt fallback validation (#429)
* Fix SDK document ordering and schema-independent root validation
* Make serialization schema test compatible with Pydantic 2.0
* Clarify reduced validation coverage when JSON Schema is skipped
* Use document_data fixture in version serialization tests
---------
Co-authored-by: JB Onofré <jb...@apache.org>
---
.../dbt/tests/__snapshots__/test_msi_to_ossie.ambr | 8 +-
.../dbt/tests/__snapshots__/test_ossie_to_msi.ambr | 2 +-
converters/dbt/tests/test_msi_to_ossie.py | 1 +
converters/orionbelt/README.md | 16 ++-
converters/orionbelt/src/ossie_orionbelt/cli.py | 12 +-
.../orionbelt/src/ossie_orionbelt/validation.py | 31 +++-
.../tests/test_ossie_converter_ontology.py | 2 +
.../orionbelt/tests/test_ossie_validation.py | 156 +++++++++++++++++++++
converters/sigma/tests/test_roundtrip.py | 1 +
converters/wisdom/tests/test_wisdom_to_ossie.py | 1 +
python/README.md | 3 +-
python/src/ossie/models.py | 10 +-
python/tests/test_models.py | 54 +++++++
13 files changed, 282 insertions(+), 15 deletions(-)
diff --git a/converters/dbt/tests/__snapshots__/test_msi_to_ossie.ambr b/converters/dbt/tests/__snapshots__/test_msi_to_ossie.ambr
index 88d1f9b8..a795f9ae 100644
--- a/converters/dbt/tests/__snapshots__/test_msi_to_ossie.ambr
+++ b/converters/dbt/tests/__snapshots__/test_msi_to_ossie.ambr
@@ -17,6 +17,7 @@
# under the License.
# name: TestMetricConversion.test_derived_metric_nested
'''
+ version: 0.2.0.dev0
name: semantic_model
datasets:
- name: orders
@@ -63,12 +64,12 @@
dialects:
- dialect: ANSI_SQL
expression: (SUM(orders.amount) - SUM(orders.cost_amount)) - SUM(orders.expense_amount)
- version: 0.2.0.dev0
'''
# ---
# name: TestMetricConversion.test_ratio_metric_inlines_sub_expressions
'''
+ version: 0.2.0.dev0
name: semantic_model
datasets:
- name: orders
@@ -101,12 +102,12 @@
- dialect: ANSI_SQL
expression: (SUM(orders.amount)) / (SUM(CASE WHEN orders.order_id IS NOT NULL
THEN 1 ELSE 0 END))
- version: 0.2.0.dev0
'''
# ---
# name: TestMetricFilterFlattening.test_metric_and_measure_filters_combined_with_and
'''
+ version: 0.2.0.dev0
name: semantic_model
datasets:
- name: orders
@@ -124,12 +125,12 @@
- dialect: ANSI_SQL
expression: SUM(CASE WHEN (status = 'paid') AND (region = 'intl') THEN orders.amount
END)
- version: 0.2.0.dev0
'''
# ---
# name: TestRelationshipConversion.test_three_datasets_produce_all_pairs
'''
+ version: 0.2.0.dev0
name: semantic_model
datasets:
- name: users_a
@@ -182,7 +183,6 @@
- user_id
to_columns:
- user_id
- version: 0.2.0.dev0
'''
# ---
diff --git a/converters/dbt/tests/__snapshots__/test_ossie_to_msi.ambr b/converters/dbt/tests/__snapshots__/test_ossie_to_msi.ambr
index 7b89427a..103c3889 100644
--- a/converters/dbt/tests/__snapshots__/test_ossie_to_msi.ambr
+++ b/converters/dbt/tests/__snapshots__/test_ossie_to_msi.ambr
@@ -17,6 +17,7 @@
# under the License.
# name: TestOssieToMSIRoundTrip.test_ossie_to_msi_to_ossie_preserves_structure
'''
+ version: 0.2.0.dev0
name: semantic_model
datasets:
- name: orders
@@ -56,7 +57,6 @@
dialects:
- dialect: ANSI_SQL
expression: SUM(orders.amount)
- version: 0.2.0.dev0
'''
# ---
diff --git a/converters/dbt/tests/test_msi_to_ossie.py b/converters/dbt/tests/test_msi_to_ossie.py
index fe9742ad..d5f9f38b 100644
--- a/converters/dbt/tests/test_msi_to_ossie.py
+++ b/converters/dbt/tests/test_msi_to_ossie.py
@@ -1412,6 +1412,7 @@ class TestOssieJsonSerialization:
parsed = json.loads(result.to_ossie_json())
assert parsed["version"] == "0.2.0.dev0"
+ assert next(iter(parsed)) == "version"
assert "semantic_model" not in parsed
assert parsed["name"] == "my_project"
diff --git a/converters/orionbelt/README.md b/converters/orionbelt/README.md
index daefed37..a038ce85 100644
--- a/converters/orionbelt/README.md
+++ b/converters/orionbelt/README.md
@@ -87,7 +87,7 @@ from ossie_orionbelt import OBMLtoOssie, OssietoOBML, validate_ossie
obml = yaml.safe_load(open("model.obml.yaml"))
ossie = OBMLtoOssie(obml, "sales", "Sales model").convert()
result = validate_ossie(ossie)
-assert result.valid
+assert result.schema_validation_performed and result.valid
obml_again = OssietoOBML(ossie).convert()
```
@@ -111,6 +111,20 @@ of the Ossie metrics with a warning and kept whole in the model-level extension
definition, and every field its OBML column name, so Ossie to OBML restores the
original model instead of re-parsing the SQL.
+`validate_ossie` checks JSON Schema conformance, unique names, and relationship
+references. If the schema file or `jsonschema` package is unavailable, it emits
+a warning and reports `JSON Schema: skipped`. The result's
+`schema_validation_performed` flag is `False` in that case.
+
+This fallback provides partial validation: it rejects non-object roots and
+legacy `semantic_model` wrappers, and checks unique names and references in
+entries it can traverse. Required fields and field types are not checked, so
+documents missing `name` or `datasets`, with malformed `datasets`, or with a
+dataset missing `source` can still return `valid=True`. That value means only
+that the checks performed found no errors. Require both
+`schema_validation_performed` and `valid`, as above, when structural validity
+is required.
+
## Vendor extensions
Ossie `custom_extensions` carry vendor-tagged payloads. This converter:
diff --git a/converters/orionbelt/src/ossie_orionbelt/cli.py b/converters/orionbelt/src/ossie_orionbelt/cli.py
index 73283218..7646e721 100644
--- a/converters/orionbelt/src/ossie_orionbelt/cli.py
+++ b/converters/orionbelt/src/ossie_orionbelt/cli.py
@@ -78,7 +78,13 @@ def _report_validation(label: str, result: dict[str, Any], validate_fn: Any) ->
for line in vr.summary_lines():
print(line, file=sys.stderr)
if vr.valid:
- print(f"{label} is valid", file=sys.stderr)
+ if vr.schema_validation_performed:
+ print(f"{label} is valid", file=sys.stderr)
+ else:
+ print(
+ f"{label} passed available checks (JSON Schema validation skipped)",
+ file=sys.stderr,
+ )
return False
print(f"{label} has validation errors", file=sys.stderr)
return True
@@ -136,7 +142,9 @@ def main(argv: list[str] | None = None) -> int:
"-o", "--output", required=True, metavar="FILE", help="Path for output Ossie YAML"
)
o2s.add_argument(
- "--ontology", action="store_true", help="Emit an Ossie ontology document instead of core-spec"
+ "--ontology",
+ action="store_true",
+ help="Emit an Ossie ontology document instead of core-spec",
)
o2s.add_argument(
"--model-name", default="semantic_model", metavar="NAME", help="Ossie semantic model name"
diff --git a/converters/orionbelt/src/ossie_orionbelt/validation.py b/converters/orionbelt/src/ossie_orionbelt/validation.py
index b4dbe6f1..cfce67c0 100644
--- a/converters/orionbelt/src/ossie_orionbelt/validation.py
+++ b/converters/orionbelt/src/ossie_orionbelt/validation.py
@@ -70,12 +70,14 @@ class ValidationResult:
def __init__(self, format_name: str = "OBML") -> None:
self.format_name = format_name
+ self.schema_validation_performed = False
self.schema_errors: list[str] = []
self.semantic_errors: list[str] = []
self.semantic_warnings: list[str] = []
@property
def valid(self) -> bool:
+ """Whether the checks that ran found no errors; skipped checks may leave gaps."""
return not self.schema_errors and not self.semantic_errors
def summary_lines(self) -> list[str]:
@@ -84,8 +86,10 @@ class ValidationResult:
lines.append(f" JSON Schema: {len(self.schema_errors)} error(s)")
for e in self.schema_errors:
lines.append(f" - {e}")
- else:
+ elif self.schema_validation_performed:
lines.append(" JSON Schema: ✓ valid")
+ else:
+ lines.append(" JSON Schema: skipped")
if self.semantic_errors:
lines.append(f" Semantic: {len(self.semantic_errors)} error(s)")
for e in self.semantic_errors:
@@ -135,6 +139,7 @@ def _validate_json_schema(
for error in sorted(validator.iter_errors(data), key=lambda e: list(e.absolute_path)):
path = ".".join(str(p) for p in error.absolute_path) or "(root)"
result.schema_errors.append(f"[{path}] {error.message}")
+ result.schema_validation_performed = True
# ── OBML Validation ──────────────────────────────────────────────────────
@@ -200,6 +205,16 @@ def validate_ossie(ossie_dict: dict[str, Any], schema_path: Path | None = None)
(Draft 2020-12)
2. **Unique names** — datasets, fields, metrics, relationships
3. **References** — relationship from/to reference existing datasets
+
+ If the schema file or ``jsonschema`` package is unavailable, schema validation
+ is skipped with a warning and ``schema_validation_performed`` is False.
+ This fallback has reduced coverage: it rejects non-object roots and legacy
+ ``semantic_model`` wrappers, and checks names and references in the entries
+ it can traverse. It does not check required fields or field types; missing
+ ``name`` or ``datasets``, malformed ``datasets``, and missing dataset ``source``
+ can pass. ``valid=True`` only means the checks that ran found no errors.
+ Require both ``schema_validation_performed`` and ``valid`` to establish
+ structural validity as well as passing the semantic checks.
"""
result = ValidationResult("Ossie")
@@ -207,13 +222,19 @@ def validate_ossie(ossie_dict: dict[str, Any], schema_path: Path | None = None)
_validate_json_schema(ossie_dict, schema_path or _OSSIE_SCHEMA_PATH, result, draft="draft2020")
# The semantic checks below assume a well-formed structure (lists of dicts).
- # JSON Schema validation above already reports structural errors, so guard
- # every level here rather than raising on malformed input.
+ # Guard every level rather than raising on malformed input, even when
+ # JSON Schema validation is unavailable.
def _as_dict_list(value: Any) -> list[dict[str, Any]]:
return [item for item in value if isinstance(item, dict)] if isinstance(value, list) else []
- # Legacy wrappers are schema errors, not model contents to traverse.
- if not isinstance(ossie_dict, dict) or "semantic_model" in ossie_dict:
+ if not isinstance(ossie_dict, dict):
+ result.semantic_errors.append("[INVALID_DOCUMENT] Ossie document must be an object")
+ return result
+ if "semantic_model" in ossie_dict:
+ result.semantic_errors.append(
+ "[LEGACY_WRAPPER] Legacy 'semantic_model' wrappers are not supported; "
+ "place model properties at the document root"
+ )
return result
model = ossie_dict
diff --git a/converters/orionbelt/tests/test_ossie_converter_ontology.py b/converters/orionbelt/tests/test_ossie_converter_ontology.py
index 93068895..2ed29523 100644
--- a/converters/orionbelt/tests/test_ossie_converter_ontology.py
+++ b/converters/orionbelt/tests/test_ossie_converter_ontology.py
@@ -132,6 +132,8 @@ class TestOntologyStructure:
result = conv.validate_ossie_ontology(doc)
assert result.valid, result.semantic_errors
assert not result.semantic_errors
+ assert not result.schema_validation_performed
+ assert result.summary_lines()[0] == " JSON Schema: skipped"
def test_one_to_one_multiplicity(self) -> None:
obml = {
diff --git a/converters/orionbelt/tests/test_ossie_validation.py b/converters/orionbelt/tests/test_ossie_validation.py
new file mode 100644
index 00000000..802a2e6a
--- /dev/null
+++ b/converters/orionbelt/tests/test_ossie_validation.py
@@ -0,0 +1,156 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import sys
+
+import pytest
+
+from ossie_orionbelt.cli import _report_validation
+from ossie_orionbelt.validation import _OSSIE_SCHEMA_PATH, validate_ossie
+
+
+@pytest.fixture(params=["available", "missing_file", "missing_package"])
+def schema_mode(request):
+ return request.param
+
+
+@pytest.fixture
+def schema_path(schema_mode, tmp_path, monkeypatch):
+ if schema_mode == "missing_file":
+ return tmp_path / "missing-schema.json"
+ if schema_mode == "missing_package":
+ monkeypatch.setitem(sys.modules, "jsonschema", None)
+ return _OSSIE_SCHEMA_PATH
+
+
+@pytest.mark.parametrize(
+ "wrapper",
+ [None, [], {}, [{"name": "legacy", "datasets": []}]],
+)
+@pytest.mark.parametrize("include_root_model", [False, True])
+def test_legacy_wrapper_is_always_invalid(schema_path, wrapper, include_root_model):
+ document = {"version": "0.2.0.dev0", "semantic_model": wrapper}
+ if include_root_model:
+ document.update(name="m", datasets=[{"name": "t", "source": "a.b.c"}])
+
+ result = validate_ossie(document, schema_path=schema_path)
+
+ assert not result.valid
+ assert any("[LEGACY_WRAPPER]" in error for error in result.semantic_errors)
+
+
+@pytest.mark.parametrize("document", [None, [], "not a mapping", 42])
+def test_non_mapping_document_is_always_invalid(schema_path, document):
+ result = validate_ossie(document, schema_path=schema_path)
+
+ assert not result.valid
+ assert any("[INVALID_DOCUMENT]" in error for error in result.semantic_errors)
+
+
+def test_valid_flat_document_can_be_checked_without_schema(schema_path, schema_mode):
+ document = {
+ "version": "0.2.0.dev0",
+ "name": "m",
+ "datasets": [{"name": "t", "source": "a.b.c"}],
+ }
+
+ result = validate_ossie(document, schema_path=schema_path)
+
+ assert result.valid
+ assert result.schema_validation_performed == (schema_mode == "available")
+ expected_status = "✓ valid" if schema_mode == "available" else "skipped"
+ assert result.summary_lines()[0] == f" JSON Schema: {expected_status}"
+ if schema_mode != "available":
+ assert result.semantic_warnings
+
+
+@pytest.mark.parametrize(
+ "model",
+ [
+ {"name": "m", "datasets": "not a list"},
+ {"name": "m"},
+ {"datasets": [{"name": "t", "source": "a.b.c"}]},
+ {"name": "m", "datasets": [{"name": "t"}]},
+ {"name": "m", "datasets": ["not an object"]},
+ ],
+ ids=["datasets_type", "datasets_missing", "name_missing", "source_missing", "dataset_type"],
+)
+def test_structural_validation_coverage_is_reported(schema_path, schema_mode, model):
+ result = validate_ossie({"version": "0.2.0.dev0", **model}, schema_path=schema_path)
+
+ assert not result.semantic_errors
+ if schema_mode == "available":
+ assert result.schema_validation_performed
+ assert not result.valid
+ assert result.schema_errors
+ assert result.summary_lines()[0] == f" JSON Schema: {len(result.schema_errors)} error(s)"
+ else:
+ assert not result.schema_validation_performed
+ assert result.valid
+ assert not result.schema_errors
+ assert result.semantic_warnings
+ assert result.summary_lines()[0] == " JSON Schema: skipped"
+
+
+@pytest.mark.parametrize("error_code", ["DUPLICATE_DATASET", "UNKNOWN_DATASET_REF"])
+def test_semantic_checks_still_run_without_schema(schema_path, error_code):
+ document = {
+ "version": "0.2.0.dev0",
+ "name": "m",
+ "datasets": [{"name": "t", "source": "a.b.c"}],
+ }
+ if error_code == "DUPLICATE_DATASET":
+ document["datasets"].append({"name": "t", "source": "a.b.d"})
+ else:
+ document["relationships"] = [
+ {
+ "name": "r",
+ "from": "t",
+ "to": "missing",
+ "from_columns": ["id"],
+ "to_columns": ["id"],
+ }
+ ]
+
+ result = validate_ossie(document, schema_path=schema_path)
+
+ assert not result.valid
+ assert any(f"[{error_code}]" in error for error in result.semantic_errors)
+
+
+@pytest.mark.parametrize("has_name", [True, False])
+def test_cli_reports_validation_coverage(schema_path, schema_mode, has_name, capsys):
+ document = {"version": "0.2.0.dev0", "datasets": [{"name": "t", "source": "a.b.c"}]}
+ if has_name:
+ document["name"] = "m"
+
+ has_errors = _report_validation(
+ "Ossie output", document, lambda doc: validate_ossie(doc, schema_path=schema_path)
+ )
+ stderr = capsys.readouterr().err
+
+ if schema_mode != "available":
+ assert not has_errors
+ assert "JSON Schema: skipped" in stderr
+ assert "passed available checks (JSON Schema validation skipped)" in stderr
+ assert "Ossie output is valid" not in stderr
+ elif has_name:
+ assert not has_errors
+ assert "Ossie output is valid" in stderr
+ else:
+ assert has_errors
+ assert "Ossie output has validation errors" in stderr
diff --git a/converters/sigma/tests/test_roundtrip.py b/converters/sigma/tests/test_roundtrip.py
index 4e5a0905..8f2361cc 100644
--- a/converters/sigma/tests/test_roundtrip.py
+++ b/converters/sigma/tests/test_roundtrip.py
@@ -39,6 +39,7 @@ def test_sigma_osi_sigma_roundtrip_through_yaml_serialization(fixture_name):
yaml_text = document.to_ossie_yaml()
serialized = yaml.safe_load(yaml_text)
+ assert next(iter(serialized)) == "version"
assert serialized["name"] == spec["name"]
assert "semantic_model" not in serialized
assert "dialects" not in serialized
diff --git a/converters/wisdom/tests/test_wisdom_to_ossie.py b/converters/wisdom/tests/test_wisdom_to_ossie.py
index e08c4d9f..9526689a 100644
--- a/converters/wisdom/tests/test_wisdom_to_ossie.py
+++ b/converters/wisdom/tests/test_wisdom_to_ossie.py
@@ -158,6 +158,7 @@ def test_stale_measure_is_kept_with_warning(result, model):
def test_output_round_trips_through_ossie_yaml(result):
serialized = yaml.safe_load(result.output.to_ossie_yaml())
+ assert next(iter(serialized)) == "version"
assert serialized["name"] == "Sample Sales"
assert "semantic_model" not in serialized
document = OssieDocument.model_validate(serialized)
diff --git a/python/README.md b/python/README.md
index 88a3b7f4..bb0ea7ba 100644
--- a/python/README.md
+++ b/python/README.md
@@ -25,7 +25,8 @@ Each `OssieDocument` is one semantic model: `name`, `datasets`, `relationships`,
and `metrics` sit at the root alongside `version`.
Construct documents with `OssieDocument(name="sales", datasets=[...])` and access
their datasets as `document.datasets`. JSON and YAML serialization use the same
-flat shape. The former `semantic_model` wrapper is rejected. Unwrap old
+flat shape, with `version` serialized first when included. The former
+`semantic_model` wrapper is rejected. Unwrap old
single-model documents and split multi-model documents into separate files,
preserving model contents and setting `version` in each file before loading them.
diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py
index a31b8b1a..43dbab86 100644
--- a/python/src/ossie/models.py
+++ b/python/src/ossie/models.py
@@ -19,7 +19,7 @@ from enum import Enum
from typing import Any, Optional, Union
import yaml
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, SerializerFunctionWrapHandler, model_serializer
class OssieDialect(str, Enum):
@@ -215,6 +215,14 @@ class OssieDocument(OssieSemanticModel):
version: str = "0.2.0.dev0"
+ @model_serializer(mode="wrap")
+ def _serialize_document(self, handler: SerializerFunctionWrapHandler):
+ # Omit a custom return type to retain Pydantic's serialization schema.
+ data = handler(self)
+ if "version" in data:
+ return {"version": data.pop("version"), **data}
+ return data
+
def to_ossie_yaml(self, **kwargs: Any) -> str:
"""Serialize to Ossie-compliant YAML (uses field aliases and excludes None values)."""
data = self.model_dump(by_alias=True, exclude_none=True, mode="json", **kwargs)
diff --git a/python/tests/test_models.py b/python/tests/test_models.py
index f809a00e..80b55098 100644
--- a/python/tests/test_models.py
+++ b/python/tests/test_models.py
@@ -146,6 +146,60 @@ def test_document_serialization_preserves_flat_model_and_metadata(
assert OssieDocument.model_validate(serialized) == document
+@pytest.mark.parametrize(
+ "serialize", ["model_dump", "model_dump_json", "to_ossie_yaml", "to_ossie_json"]
+)
+def test_document_serialization_puts_version_first(
+ serialize: str, document_data: dict
+) -> None:
+ data = document_data
+ document = OssieDocument.model_validate(data)
+
+ serialized = getattr(document, serialize)()
+ if isinstance(serialized, str):
+ serialized = yaml.safe_load(serialized)
+
+ assert next(iter(serialized)) == "version"
+ assert serialized["version"] == data["version"]
+
+
+@pytest.mark.parametrize(
+ "serialize", ["model_dump", "model_dump_json", "to_ossie_yaml", "to_ossie_json"]
+)
+@pytest.mark.parametrize(
+ "options",
+ [
+ {"exclude": {"version"}},
+ {"include": {"name", "datasets"}},
+ {"exclude_defaults": True},
+ {"exclude_unset": True},
+ ],
+)
+def test_document_serialization_can_omit_version(
+ serialize: str, options: dict, document_data: dict
+) -> None:
+ data = document_data
+ del data["version"]
+ document = OssieDocument.model_validate(data)
+
+ serialized = getattr(document, serialize)(**options)
+ if isinstance(serialized, str):
+ serialized = yaml.safe_load(serialized)
+
+ assert "version" not in serialized
+ assert serialized["name"] == data["name"]
+
+
+def test_document_serialization_schema_preserves_model_fields() -> None:
+ schema = OssieDocument.model_json_schema(mode="serialization")
+
+ assert schema["properties"]["version"]["type"] == "string"
+ assert schema["properties"]["datasets"]["type"] == "array"
+ # Early Pydantic 2.x versions also require defaulted fields in serialization schemas.
+ assert {"name", "datasets"} <= set(schema["required"])
+ assert schema["additionalProperties"] is False
+
+
@pytest.mark.parametrize(
"legacy_value",
[