You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ossie.apache.org by kb...@apache.org on 2026/07/15 23:14:12 UTC
(ossie) branch main updated: [OSI][Omni] Add Omni semantic model converter (#175)
This is an automated email from the ASF dual-hosted git repository.
khush-bhatia pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/ossie.git
The following commit(s) were added to refs/heads/main by this push:
new 309acbf [OSI][Omni] Add Omni semantic model converter (#175)
309acbf is described below
commit 309acbf2c0a29716514ad11168e9131bc12c470b
Author: Jamie Davidson <jd...@gmail.com>
AuthorDate: Wed Jul 15 16:14:06 2026 -0700
[OSI][Omni] Add Omni semantic model converter (#175)
* Add Omni bidirectional converter
Adds converters/omni: an offline OSI <-> Omni semantic model converter,
filling the Omni spoke in the hub-and-spoke architecture.
- Export (osi-omni export): OSI -> Omni model files (views/*.view.yaml,
relationships.yaml, a generated topic, model.yaml when stashed).
- Import (osi-omni import): Omni -> OSI. Omni-only features are preserved
in custom_extensions[OMNI], so Omni -> OSI -> Omni is lossless.
- Unified osi-omni CLI plus a string-in / files-dict-out Python API.
- Example-based unit tests, fixture round-trips (incl. TPC-DS), and
Hypothesis property-based round-trip tests with a seeded-random fallback.
- Adds the OMNI vendor row to converters/index.md.
Co-Authored-By: Claude Fable 5 <no...@anthropic.com>
* Handle real-world Omni model layouts in the OSI converter
Testing against production Omni instances surfaced model shapes the
fixtures did not cover; importing all of them now round-trips losslessly
(verified on a 3,498-file and a 427-file production model plus the
Ecommerce demo model, all validating against the OSI spec):
- Resolve schema-qualified view names (schema__table) from the
"# Reference this view as" header, accept per-schema folder layouts,
and preserve original file paths through a round trip.
- Stash-and-restore instead of erroring: extends-only views, non-equi/
range joins, joins touching query/extends views, and fields or
measures using Omni template ({{...}}) syntax.
- Preserve explicit Omni defaults verbatim (join_type, relationship_type,
redundant table_name, self-named sql), non-canonical on_sql (aliases,
casing, spacing), ${view.field} compound keys, empty-string metadata,
and timeframes: [].
- Accept real Omni identifiers (_fivetran_id, trailing underscores,
camelCase); collision checks stay case-insensitive.
- Quote source parts that need it ("Omni Views".upload), tag date values
in the JSON stash, and suffix duplicate generated relationship names.
Co-Authored-By: Claude Fable 5 <no...@anthropic.com>
* Catch sqlglot TokenError in the SQL validator
An expression that cannot be tokenized (e.g. templated SQL) crashed the
validator instead of being reported as a finding.
Co-Authored-By: Claude Fable 5 <no...@anthropic.com>
---------
Co-authored-by: Claude Fable 5 <no...@anthropic.com>
---
converters/README.md | 1 +
converters/omni/README.md | 168 +++++
converters/omni/pyproject.toml | 31 +
converters/omni/src/osi_omni/__init__.py | 16 +
converters/omni/src/osi_omni/_common.py | 405 ++++++++++++
converters/omni/src/osi_omni/cli.py | 101 +++
converters/omni/src/osi_omni/omni_to_osi.py | 701 +++++++++++++++++++++
converters/omni/src/osi_omni/osi_to_omni.py | 609 ++++++++++++++++++
converters/omni/tests/_roundtrip_helpers.py | 361 +++++++++++
converters/omni/tests/_util.py | 116 ++++
converters/omni/tests/conftest.py | 6 +
.../fixtures/fixtureA_omni/relationships.yaml | 4 +
.../fixtures/fixtureA_omni/topics/sales.topic.yaml | 5 +
.../fixtureA_omni/views/customer.view.yaml | 7 +
.../fixtures/fixtureA_omni/views/orders.view.yaml | 32 +
converters/omni/tests/fixtures/fixtureA_osi.yaml | 83 +++
.../omni/tests/fixtures/fixtureB_omni/model.yaml | 10 +
.../fixtures/fixtureB_omni/relationships.yaml | 12 +
.../fixtureB_omni/topics/order_analysis.topic.yaml | 15 +
.../fixtureB_omni/views/order_items.view.yaml | 46 ++
.../fixtures/fixtureB_omni/views/orders.view.yaml | 6 +
.../fixtures/fixtureB_omni/views/users.view.yaml | 9 +
.../tests/fixtures/tpcds_omni/relationships.yaml | 16 +
.../topics/tpcds_retail_model.topic.yaml | 11 +
.../fixtures/tpcds_omni/views/customer.view.yaml | 27 +
.../fixtures/tpcds_omni/views/date_dim.view.yaml | 53 ++
.../tests/fixtures/tpcds_omni/views/item.view.yaml | 33 +
.../fixtures/tpcds_omni/views/store.view.yaml | 32 +
.../tpcds_omni/views/store_sales.view.yaml | 90 +++
converters/omni/tests/test_omni_to_osi.py | 376 +++++++++++
converters/omni/tests/test_osi_to_omni.py | 334 ++++++++++
converters/omni/tests/test_real_world_layout.py | 170 +++++
converters/omni/tests/test_roundtrip.py | 65 ++
converters/omni/tests/test_roundtrip_properties.py | 83 +++
validation/validate.py | 6 +-
35 files changed, 4037 insertions(+), 3 deletions(-)
diff --git a/converters/README.md b/converters/README.md
index 9dd4f98..8daf5d0 100644
--- a/converters/README.md
+++ b/converters/README.md
@@ -73,6 +73,7 @@ The Ossie specification currently defines extensions for the following vendors:
| `SALESFORCE` | Salesforce / Tableau semantic layer |
| `DBT` | dbt semantic models |
| `DATABRICKS` | Databricks semantic layer |
+| `OMNI` | Omni semantic model |
Each vendor may define custom extensions (via the `custom_extensions` field in the Ossie spec) to carry vendor-specific metadata that does not have an equivalent in the core specification.
diff --git a/converters/omni/README.md b/converters/omni/README.md
new file mode 100644
index 0000000..c1029cc
--- /dev/null
+++ b/converters/omni/README.md
@@ -0,0 +1,168 @@
+# OSI <-> Omni converter
+
+Bidirectional, offline conversion between an OSI semantic model and
+[Omni](https://docs.omni.co/modeling) semantic model files. No Omni connection
+required.
+
+An Omni model is a *directory* of YAML files rather than a single document, so
+this converter maps one OSI YAML document to/from the Omni model layout:
+
+```
+model.yaml # model-wide config (restored verbatim from a prior import)
+relationships.yaml # top-level list of join definitions
+views/<name>.view.yaml # one per OSI dataset
+topics/<name>.topic.yaml # one generated topic per OSI model (or restored originals)
+```
+
+Import also accepts the layout Omni's API/IDE emits (`omni models yaml-get`,
+git sync): view/topic files in per-schema or arbitrary folders
+(`DELIGHTED/response.view`), bare `.view`/`.topic` suffixes, and
+schema-qualified view names (`delighted__response`) taken from each file's
+`# Reference this view as ...` header (falling back to the file's basename).
+Original file paths are preserved through a round trip.
+
+- **Export** (`osi-omni export`): OSI -> Omni files. Datasets become views,
+ relationships become `relationships.yaml` joins, metrics become measures on
+ the view they reference, and the model becomes a topic (rooted at the
+ fact/FK-sink dataset, or `--base-view`).
+- **Import** (`osi-omni import`): Omni files -> OSI. Omni features OSI has no
+ native field for are preserved in `custom_extensions[OMNI]`, so
+ **Omni -> OSI -> Omni is lossless**.
+
+On **export** (OSI -> Omni), OSI features with no Omni slot -- `unique_keys`,
+relationship `ai_context`, dataset-level `ai_context` synonyms, model-level
+`ai_context` synonyms/examples, foreign-vendor `custom_extensions`, fields and
+metrics without a usable dialect -- are **dropped with a warning**. On
+**import** (Omni -> OSI), Omni-only features (formats, timeframes, hidden/tags,
+topic curation, the model file, ...) are instead **preserved** in
+`custom_extensions[OMNI]`. Any input that breaks a
+[requirement](#requirements) **raises a `ConversionError`** -- the converter
+never silently drops a field or produces an invalid result.
+
+## Installation
+
+```bash
+pip install osi-omni # once published to PyPI
+# or, from a checkout of this directory:
+pip install -e .
+```
+
+The only runtime dependency is `PyYAML`. Python 3.9+.
+
+## Usage
+
+### Command line
+
+```bash
+osi-omni export -i model.yaml -o omni_model/ [--base-view orders] [--dialect SNOWFLAKE]
+osi-omni import -i omni_model/ [-o model.yaml] [--name my_model] [--topic orders]
+```
+
+`export` writes the Omni files into the `-o` directory. `import` reads a model
+directory (the [local-editor / git layout](https://docs.omni.co/guides/modeling/local-development),
+`.yaml`-suffixed or bare `.view`/`.topic` names both work); with no `-o` the
+OSI YAML goes to stdout. `--base-view` picks the dataset the generated topic is
+rooted at (default: the FK-sink dataset). `--topic` picks which topic's
+description/AI context map onto the OSI model when there are several.
+
+### Python API
+
+```python
+from osi_omni import convert_osi_to_omni, convert_omni_to_osi
+
+files = convert_osi_to_omni(osi_yaml_str) # -> {relative filename: YAML str}
+osi_yaml = convert_omni_to_osi(files) # {relative filename: YAML str} -> str
+```
+
+## Mapping
+
+Each row maps in both directions; the **Notes** flag where a behavior is
+specific to **export** (OSI -> Omni) or **import** (Omni -> OSI).
+
+| OSI | Omni | Notes |
+|---|---|---|
+| `semantic_model.name` | topic file name | Import: the mapped topic's name (override with `--name`). |
+| `model.description` / `ai_context.instructions` | topic `description` / `ai_context` | Import: taken from the sole topic, or `--topic`. |
+| dataset | `views/<name>.view.yaml` | Import: a stashed original path (`DELIGHTED/response.view`) is restored on export. |
+| `dataset.source` `catalog.schema.table` / `schema.table` | view `catalog` + `schema` + `table_name` | `table_name` left implicit when it matches the file name; a part that is not a plain identifier is double-quoted (`"Omni Views".upload`). |
+| `dataset.source` `SELECT ...` | view `sql:` | A SQL-defined view. |
+| `dataset.description` / `ai_context.instructions` | view `description` / `ai_context` | |
+| `dataset.primary_key` (single) | `primary_key: true` on the matching dimension | Export: a key column no field covers becomes a hidden dimension. |
+| `dataset.primary_key` (composite) | view `custom_compound_primary_key_sql` | Import: `${view.field}` entries resolve to plain field names (original list stashed); multiple `primary_key: true` dimensions also form a composite key. |
+| field | dimension | Export: an already-valid Omni identifier (incl. `_fivetran_id`, `..._day_`, camelCase) passes through; anything else sanitizes to `[a-z][a-z0-9_]*`. A case-insensitive collision is an error. |
+| `field.expression` | dimension `sql` | Export: a bare column named like the field emits `{}` (the schema-layer default); import translates `${field}`/`${view.field}`/`${TABLE}.col` references and stashes the original `sql`. |
+| `field.dimension.is_time` | dimension `timeframes` | Export: the Omni default list; import stashes the exact list. |
+| `field.label` / `description` | `label` / `description` | |
+| `field.ai_context` synonyms / instructions | `synonyms` / `ai_context` | |
+| relationship | `relationships.yaml` entry | `from`(many) -> `join_from_view`, `to`(one) -> `join_to_view`, columns -> `on_sql` equi-join. Declared `join_type`/`relationship_type` (even Omni defaults) and any `on_sql` the rebuild would not reproduce (aliases, `and` casing, spacing) are stashed verbatim. |
+| `relationship.name` | -- | Regenerated as `<from>_to_<to>` on import (suffixed `_2`, `_3`, ... when several joins share a view pair). |
+| metric | measure on the view its expression references (else the base view) | `AGG(view.field)` <-> `aggregate_type` + `sql: ${field}`; `COUNT(*)` <-> `aggregate_type: count`; anything else <-> a raw-`sql` measure. |
+| `metric.description` / `ai_context` | measure `description` / `synonyms` / `ai_context` | |
+| `custom_extensions[OMNI]` | everything Omni-only | Import stashes; export restores -- keeping `Omni -> OSI -> Omni` lossless. |
+
+**Stashed on import** (and restored on export): the model file (verbatim),
+topics (verbatim, minus the natively-mapped description/AI context), original
+file paths, view extras (`label`, `hidden`, `tags`, view-level `filters:`,
+...), dimension extras (`format`, `group_label`, exact `timeframes`, original
+`sql`, ...), present-but-empty metadata (`description: ''`), join extras
+(declared `join_type`/`relationship_type`, `reversible`, `where_sql`, aliases,
+non-canonical `on_sql`), joins OSI cannot represent (non-equi/range joins,
+joins touching a query or extends-only view -- restored at their original
+positions), fields/measures whose sql uses Omni template (`{{...}}`) syntax,
+non-reconstructible measures (filtered, `percentile`/`list`/`*_distinct_on`,
+raw-SQL), and files with no OSI form (query views, extends-only views,
+unrecognized files).
+
+**Expression dialects**: Omni SQL is the SQL of the model's database
+connection, and the OSI dialect enum has no `OMNI` entry -- so import emits
+`ANSI_SQL` expressions, and export prefers `ANSI_SQL` with `--dialect`
+prepending a warehouse dialect (e.g. `SNOWFLAKE` for a Snowflake-backed Omni
+model). A field/metric with neither is dropped with a warning.
+
+## Requirements
+
+Conversion raises a `ConversionError` (rather than guessing or emitting
+something invalid) when an input breaks one of these:
+
+- a dataset `source` has no schema part (Omni views require `schema`);
+- the relationship graph gives no unambiguous base view for the generated topic
+ (multiple FK sinks or a cycle) and `--base-view` is not given;
+- two names sanitize to the same Omni identifier, case-insensitively (never
+ silently merged); two view files resolve to the same canonical view name;
+- a measure has an unknown `aggregate_type`; an import directory has no
+ convertible view files; the input YAML is malformed.
+
+## Notes and limitations
+
+- Exported `on_sql` references columns as `${view.column}`. Omni resolves these
+ against the schema layer, which auto-generates a dimension per physical
+ column, so the reference is valid even when the OSI model declares no field
+ for the column.
+- `dimension: {is_time: false}` is equivalent to omitting `dimension` and is
+ normalized away on a round trip.
+- One generated topic per OSI model; multi-path (aliased) join fan-out and
+ Omni query views, extends-only views, non-equi joins, composite topics,
+ access grants/filters, and templated filters are stash-and-restore only (no
+ OSI semantics).
+- A model containing *only* query/extends views (e.g. a pure GA4-export model)
+ has nothing to convert and is rejected.
+- OSI metric order is regrouped by view on import (order is not semantic).
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+python3 -m pytest tests/
+```
+
+Example-based unit tests plus Hypothesis property-based round-trip tests
+(`test_roundtrip_properties.py`, which fall back to a seeded-random sweep if
+`hypothesis` is not installed).
+
+## Future effort
+
+Both the OSI specification and Omni's model YAML are still evolving. As either
+side adds or changes fields, this converter will be updated to track them --
+extending the mapping and coverage in both directions (query views, composite
+topics, measure filters as first-class OSI once the spec grows a slot for
+them) to keep the conversion current.
diff --git a/converters/omni/pyproject.toml b/converters/omni/pyproject.toml
new file mode 100644
index 0000000..b4be76f
--- /dev/null
+++ b/converters/omni/pyproject.toml
@@ -0,0 +1,31 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "osi-omni"
+version = "0.2.0.dev0"
+description = "Bidirectional converter between OSI semantic models and Omni semantic model files"
+requires-python = ">=3.9"
+dependencies = [
+ "PyYAML>=6.0",
+]
+
+[project.license]
+text = "Apache-2.0"
+
+[project.optional-dependencies]
+dev = [
+ "pytest>=8.0",
+ "hypothesis>=6.0",
+]
+
+[project.scripts]
+osi-omni = "osi_omni.cli:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/osi_omni"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]
diff --git a/converters/omni/src/osi_omni/__init__.py b/converters/omni/src/osi_omni/__init__.py
new file mode 100644
index 0000000..8eab424
--- /dev/null
+++ b/converters/omni/src/osi_omni/__init__.py
@@ -0,0 +1,16 @@
+"""Bidirectional converter between OSI semantic models and Omni semantic model
+files (model.yaml / relationships.yaml / views/*.view.yaml / topics/*.topic.yaml).
+Pure offline transforms: OSI YAML string <-> {relative filename: YAML string}.
+
+ from osi_omni import convert_osi_to_omni, convert_omni_to_osi
+"""
+
+from ._common import ConversionError
+from .omni_to_osi import convert_omni_to_osi
+from .osi_to_omni import convert_osi_to_omni
+
+__all__ = [
+ "ConversionError",
+ "convert_omni_to_osi",
+ "convert_osi_to_omni",
+]
diff --git a/converters/omni/src/osi_omni/_common.py b/converters/omni/src/osi_omni/_common.py
new file mode 100644
index 0000000..571b525
--- /dev/null
+++ b/converters/omni/src/osi_omni/_common.py
@@ -0,0 +1,405 @@
+"""Shared helpers for the OSI <-> Omni converters.
+
+Both directions are pure offline YAML transforms. The cross-cutting concerns live
+here: version constants, the dialect preference order, the `custom_extensions`
+stash protocol, Omni file-name conventions, identifier sanitization, and the
+`${...}` reference translation between Omni SQL and the plain column references
+OSI expressions use.
+"""
+
+import datetime
+import json
+import re
+
+import yaml
+
+# OSI semantic model spec version this converter targets (see core-spec).
+OSI_VERSION = "0.2.0.dev0"
+
+# Vendor id used for the `custom_extensions` stash.
+VENDOR = "OMNI"
+
+# Omni SQL is the SQL of the model's database connection, so there is no OMNI
+# entry in the OSI dialect enum. Import emits ANSI_SQL; export prefers ANSI_SQL
+# and lets the caller prepend a warehouse dialect (e.g. SNOWFLAKE) that the
+# actual connection would accept.
+DIALECT_ANSI = "ANSI_SQL"
+
+# Bump when the shape of a stashed `data` blob changes.
+STASH_VERSION = 1
+
+# Omni model file names (the local-editor/git layout, with `.yaml` appended).
+MODEL_FILE = "model.yaml"
+RELATIONSHIPS_FILE = "relationships.yaml"
+VIEW_DIR = "views"
+TOPIC_DIR = "topics"
+
+# Omni relationship defaults (left implicit on export when they hold).
+DEFAULT_JOIN_TYPE = "always_left"
+REL_MANY_TO_ONE = "many_to_one"
+REL_ONE_TO_MANY = "one_to_many"
+
+# The timeframes Omni applies to a time dimension by default; used to represent
+# OSI `dimension.is_time` when no exact list is stashed.
+DEFAULT_TIMEFRAMES = ["raw", "date", "week", "month", "quarter", "year"]
+
+# A valid Omni identifier (view, dimension, measure, topic name). Broader than
+# the documented lowercase convention because real Omni-generated models use
+# more: leading underscores (Fivetran's `_fivetran_id`), trailing underscores
+# (truncated column names), and camelCase (JSON-flattened `..._dimensionIndex`).
+_OMNI_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+# A bare SQL identifier (single column reference), e.g. `c_name`.
+_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+# An Omni `${...}` reference: ${field}, ${view.field}, ${view.field[timeframe]},
+# ${TABLE}.column. Group 1 is the reference body (without the braces).
+_OMNI_REF_RE = re.compile(r"\$\{\s*([^}]*?)\s*\}")
+
+
+class ConversionError(Exception):
+ """Raised when an input cannot be converted."""
+
+
+def require(obj, key, what):
+ """Return `obj[key]`, or raise a clean ConversionError if it's missing/empty --
+ so malformed input surfaces as an error message rather than a raw KeyError.
+
+ Presence is tested by key (not truthiness), so a legitimately falsy value such
+ as `0` or `False` is returned; a missing key, a null, or an empty/whitespace
+ string is rejected.
+ """
+ if not isinstance(obj, dict) or key not in obj or obj[key] is None:
+ raise ConversionError(f"{what} is missing required '{key}'")
+ value = obj[key]
+ if isinstance(value, str) and not value.strip():
+ raise ConversionError(f"{what} has an empty '{key}'")
+ return value
+
+
+def require_str(obj, key, what):
+ """Like require(), but also enforce the value is a string -- so a non-string
+ scalar (e.g. a YAML number for a name or expression) raises a clean
+ ConversionError instead of crashing later in a string operation."""
+ value = require(obj, key, what)
+ if not isinstance(value, str):
+ raise ConversionError(
+ f"{what}: '{key}' must be a string, got {type(value).__name__}")
+ return value
+
+
+# PyYAML's default YAML 1.1 semantics turn bare on/off/yes/no into booleans, which
+# would corrupt Omni string values (a label "On", a week_start_day, a synonym).
+# The Loader below uses YAML 1.2 booleans (only true/false); the Dumper
+# force-quotes bool-like string tokens so the output round-trips through a 1.1
+# reader too. Same approach as the osi-databricks converter.
+class _Yaml12Loader(yaml.SafeLoader):
+ """SafeLoader with YAML 1.2 boolean semantics."""
+
+
+class _Yaml12Dumper(yaml.SafeDumper):
+ """SafeDumper with YAML 1.2 boolean semantics."""
+
+
+_YAML12_BOOL = re.compile(r"^(?:true|True|TRUE|false|False|FALSE)$")
+for _cls in (_Yaml12Loader, _Yaml12Dumper):
+ # Drop the YAML 1.1 bool resolver (yes/no/on/off/y/n) and re-add a 1.2 one.
+ _cls.yaml_implicit_resolvers = {
+ ch: [(tag, rx) for (tag, rx) in resolvers if tag != "tag:yaml.org,2002:bool"]
+ for ch, resolvers in _cls.yaml_implicit_resolvers.items()
+ }
+ _cls.add_implicit_resolver("tag:yaml.org,2002:bool", _YAML12_BOOL, list("tTfF"))
+
+
+_YAML11_BOOL_STRS = frozenset(
+ variant
+ for word in ("y", "n", "yes", "no", "on", "off", "true", "false")
+ for variant in (word, word.capitalize(), word.upper())
+)
+
+
+def _represent_str(dumper, data):
+ style = "'" if data in _YAML11_BOOL_STRS else None
+ if "\n" in data:
+ style = "|"
+ return dumper.represent_scalar("tag:yaml.org,2002:str", data, style=style)
+
+
+_Yaml12Dumper.add_representer(str, _represent_str)
+
+
+def load_yaml(text, what="input"):
+ """Parse YAML with 1.2 boolean semantics. A syntax error is surfaced as a
+ ConversionError so callers (and the CLI) get a clean message."""
+ try:
+ return yaml.load(text, Loader=_Yaml12Loader)
+ except yaml.YAMLError as e:
+ raise ConversionError(f"Invalid YAML in {what}: {e}") from e
+
+
+def dump_yaml(obj):
+ """Serialize to YAML with 1.2 boolean semantics; bool-like string tokens are
+ force-quoted so a YAML 1.1 reader of this output sees strings, not booleans."""
+ return yaml.dump(obj, Dumper=_Yaml12Dumper, sort_keys=False,
+ default_flow_style=False, allow_unicode=True)
+
+
+def is_simple_identifier(expr):
+ """True if `expr` is a single bare column reference (no operators/functions)."""
+ return isinstance(expr, str) and bool(_IDENTIFIER_RE.match(expr.strip()))
+
+
+def is_omni_name(name):
+ """True if `name` is already a valid Omni identifier."""
+ return isinstance(name, str) and bool(_OMNI_NAME_RE.match(name))
+
+
+def sanitize_name(name, what, taken):
+ """Coerce an OSI name into a valid Omni identifier.
+
+ A name that is already a valid Omni identifier passes through untouched
+ (leading/trailing underscores and camelCase are legal and occur in real
+ Omni-generated models); anything else is lowercased with every invalid
+ character run replaced by `_`. A result colliding case-insensitively with
+ one already in `taken` (a set of casefolded names) is an error rather than
+ a silent merge; the caller adds `result.lower()` to `taken`.
+ """
+ raw = str(name)
+ if _OMNI_NAME_RE.match(raw):
+ out = raw
+ else:
+ out = re.sub(r"[^a-z0-9_]+", "_", raw.lower()).strip("_")
+ if not out or not out[0].isalpha():
+ out = f"v_{out}" if out else "v"
+ if out.lower() in taken:
+ raise ConversionError(
+ f"{what} '{name}' sanitizes to '{out}', which collides with another "
+ f"name; rename it in the OSI model."
+ )
+ return out
+
+
+def view_file(view_name):
+ return f"{VIEW_DIR}/{view_name}.view.yaml"
+
+
+def topic_file(topic_name):
+ return f"{TOPIC_DIR}/{topic_name}.topic.yaml"
+
+
+# YAML parses a bare `2024-01-01` (e.g. in a topic's default_filters) into a
+# datetime.date, which JSON cannot hold; the stash tags such values so they come
+# back as dates and re-dump unquoted, keeping the round trip lossless.
+_JSON_TEMPORAL = {"date": datetime.date, "datetime": datetime.datetime}
+
+
+def _json_default(o):
+ for tag, cls in _JSON_TEMPORAL.items():
+ if type(o) is cls:
+ return {"__osi_omni__": tag, "v": o.isoformat()}
+ raise TypeError(f"Object of type {type(o).__name__} is not JSON serializable")
+
+
+def _json_object_hook(d):
+ cls = _JSON_TEMPORAL.get(d.get("__osi_omni__", ""))
+ if cls is not None and set(d) == {"__osi_omni__", "v"}:
+ return cls.fromisoformat(d["v"])
+ return d
+
+
+def read_stash(obj):
+ """Return the OMNI stash dict on an OSI object, or {} if absent.
+
+ The `_v` version marker is stripped from the returned dict.
+ """
+ for ext in (obj or {}).get("custom_extensions") or []:
+ if ext.get("vendor_name") == VENDOR:
+ data = json.loads(ext.get("data") or "{}",
+ object_hook=_json_object_hook)
+ data.pop("_v", None)
+ return data
+ return {}
+
+
+def write_stash(obj, data):
+ """Attach an OMNI `custom_extensions` entry holding `data` (a dict).
+
+ No-op when `data` is empty, so hand-authored OSI stays clean. Merges into an
+ existing OMNI entry if one is already present.
+ """
+ if not data:
+ return
+ payload = {"_v": STASH_VERSION}
+ payload.update(data)
+ blob = json.dumps(payload, default=_json_default)
+ exts = obj.setdefault("custom_extensions", [])
+ for ext in exts:
+ if ext.get("vendor_name") == VENDOR:
+ ext["data"] = blob
+ return
+ exts.append({"vendor_name": VENDOR, "data": blob})
+
+
+def foreign_vendor_extensions(obj):
+ """Return non-OMNI custom_extensions (dropped on export, with a warning)."""
+ return [
+ ext
+ for ext in (obj or {}).get("custom_extensions") or []
+ if ext.get("vendor_name") != VENDOR
+ ]
+
+
+def pick_expression(osi_expression, preferred=None):
+ """Choose the SQL string for an OSI expression.
+
+ Preference order: the caller-chosen warehouse dialect (Omni passes SQL through
+ to the connection's database, so e.g. SNOWFLAKE SQL is valid on a Snowflake-
+ backed Omni model), then ANSI_SQL. Returns None if neither is present (the
+ caller warns and skips).
+ """
+ dialects = {
+ d.get("dialect"): d.get("expression")
+ for d in (osi_expression or {}).get("dialects") or []
+ }
+ expr = None
+ if preferred:
+ expr = dialects.get(preferred)
+ if expr is None:
+ expr = dialects.get(DIALECT_ANSI)
+ if expr is not None and not isinstance(expr, str):
+ raise ConversionError(
+ f"expression must be a string, got {type(expr).__name__}")
+ return expr
+
+
+def synonyms_of(ai_context):
+ """Extract the synonyms list from an OSI ai_context (object form only)."""
+ if isinstance(ai_context, dict):
+ return list(ai_context.get("synonyms") or [])
+ return []
+
+
+def instructions_of(ai_context):
+ """The free-text part of an OSI ai_context: the string itself, or the
+ object form's `instructions`."""
+ if isinstance(ai_context, str) and ai_context.strip():
+ return ai_context
+ if isinstance(ai_context, dict):
+ text = ai_context.get("instructions")
+ if isinstance(text, str) and text.strip():
+ return text
+ return None
+
+
+# One part of a dotted source reference: double-quoted (may hold spaces/dots --
+# Omni's uploaded-CSV schema is literally `Omni Views`) or a bare name.
+_SOURCE_PARTS_RE = re.compile(r'^(?:"[^"]+"|[^".]+)(?:\.(?:"[^"]+"|[^".]+))*$')
+_SOURCE_PART_RE = re.compile(r'"([^"]+)"|([^".]+)')
+
+
+def quote_source_part(part):
+ """Quote one part of an OSI dotted source when it needs it."""
+ p = str(part)
+ return p if re.fullmatch(r"[A-Za-z0-9_$]+", p) else f'"{p}"'
+
+
+def parse_source(source, dataset_name):
+ """Split an OSI dataset `source` into Omni view placement.
+
+ Returns ("sql", sql_text) for a SELECT/WITH subquery source, or
+ ("table", catalog_or_None, schema, table) for a dotted table reference
+ (parts may be double-quoted: `"Omni Views".channel_info`).
+ Omni views require a `schema`, so a bare 1-part table name is rejected.
+ """
+ if not source or not str(source).strip():
+ raise ConversionError(f"Dataset '{dataset_name}': missing/empty 'source'")
+ s = str(source).strip()
+ if re.match(r"(?i)(select|with)\b", s):
+ return ("sql", s)
+ if not _SOURCE_PARTS_RE.match(s):
+ raise ConversionError(
+ f"Dataset '{dataset_name}': source '{source}' is not a valid dotted "
+ f"table reference or SELECT/WITH subquery"
+ )
+ parts = []
+ for m in _SOURCE_PART_RE.finditer(s):
+ quoted, bare = m.group(1), m.group(2)
+ if bare is not None and any(ch.isspace() for ch in bare):
+ raise ConversionError(
+ f"Dataset '{dataset_name}': source '{source}' is not a valid "
+ f"dotted table reference or SELECT/WITH subquery"
+ )
+ parts.append(quoted if quoted is not None else bare)
+ if len(parts) == 3:
+ return ("table", parts[0], parts[1], parts[2])
+ if len(parts) == 2:
+ return ("table", None, parts[0], parts[1])
+ raise ConversionError(
+ f"Dataset '{dataset_name}': source '{source}' has no schema part; Omni "
+ f"views require a `schema`, so use `schema.table` or `catalog.schema.table`"
+ )
+
+
+def join_source(view):
+ """Rebuild an OSI dataset `source` string from an Omni view dict."""
+ if view.get("sql") is not None:
+ return str(view["sql"]).strip()
+ schema = view.get("schema")
+ table = view.get("table_name")
+ if not schema or not table:
+ return None
+ parts = [view["catalog"], schema, table] if view.get("catalog") else [schema, table]
+ return ".".join(quote_source_part(p) for p in parts)
+
+
+def omni_sql_to_osi(sql, own_view):
+ """Translate Omni `${...}` references in a SQL string to the plain references
+ OSI expressions use. Returns (translated, changed).
+
+ - `${TABLE}.col` -> `col` (a raw column of the owning view)
+ - `${field}` -> `field` (same-view field)
+ - `${own_view.field}` -> `field` (qualified same-view field)
+ - `${other.field}` -> `other.field`
+ - `${view.field[tf]}` -> `view.field` (timeframe access has no OSI form;
+ the caller warns and stashes the original)
+ OSI has no field-vs-column distinction, so both flavors flatten to names.
+ """
+ changed = False
+
+ def repl(m):
+ nonlocal changed
+ changed = True
+ body = m.group(1)
+ if body == "TABLE":
+ return "__OSI_TABLE__" # handled below with its trailing dot
+ body = re.sub(r"\[[^\]]*\]$", "", body).strip() # drop [timeframe]
+ if "." in body:
+ head, rest = body.split(".", 1)
+ if head == own_view:
+ return rest
+ return body
+
+ out = _OMNI_REF_RE.sub(repl, sql)
+ out = out.replace("__OSI_TABLE__.", "").replace("__OSI_TABLE__", "")
+ return out, changed
+
+
+def has_timeframe_ref(sql):
+ """True if the Omni SQL contains a `${view.field[timeframe]}` reference."""
+ return bool(re.search(r"\$\{[^}]*\[[^\]]*\][^}]*\}", sql or ""))
+
+
+def osi_expr_refs_to_omni(expr, view_names):
+ """Rewrite `view.column` references in an OSI expression to Omni `${view.column}`
+ form, for the known `view_names` only -- so a genuine schema-qualified table or
+ an unrelated dotted token is left alone. Bare columns stay bare (raw columns
+ are legal in Omni SQL)."""
+ if not view_names:
+ return expr
+
+ pattern = re.compile(
+ r"(?<![\w.$])(" + "|".join(re.escape(v) for v in sorted(view_names, key=len,
+ reverse=True))
+ + r")\.([A-Za-z_][A-Za-z0-9_]*)(?![\w.])"
+ )
+ return pattern.sub(lambda m: "${" + m.group(1) + "." + m.group(2) + "}", expr)
diff --git a/converters/omni/src/osi_omni/cli.py b/converters/omni/src/osi_omni/cli.py
new file mode 100644
index 0000000..70fc7a1
--- /dev/null
+++ b/converters/omni/src/osi_omni/cli.py
@@ -0,0 +1,101 @@
+"""Command-line interface for the OSI <-> Omni converter.
+
+ osi-omni export -i model.yaml -o omni_model/ [--base-view orders] [--dialect SNOWFLAKE]
+ osi-omni import -i omni_model/ [-o model.yaml] [--name my_model] [--topic orders]
+
+`export` converts an OSI semantic model into an Omni model directory
+(model.yaml / relationships.yaml / views/*.view.yaml / topics/*.topic.yaml);
+`import` does the reverse. Import with no `-o` writes the OSI YAML to stdout;
+export always needs `-o` (a directory). Conversions that drop information emit
+warnings to stderr.
+"""
+
+import argparse
+import os
+import sys
+
+from ._common import ConversionError
+from .omni_to_osi import convert_omni_to_osi
+from .osi_to_omni import convert_osi_to_omni
+
+
+def _build_parser():
+ parser = argparse.ArgumentParser(prog="osi-omni", description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter)
+ sub = parser.add_subparsers(dest="command")
+ sub.required = True
+
+ exp = sub.add_parser("export", help="OSI semantic model -> Omni model directory")
+ exp.add_argument("-i", "--input", required=True, help="OSI YAML file")
+ exp.add_argument("-o", "--output", required=True,
+ help="output directory for the Omni model files")
+ exp.add_argument("-b", "--base-view",
+ help="dataset the generated topic is rooted at "
+ "(default: the FK-sink dataset)")
+ exp.add_argument("-d", "--dialect",
+ help="preferred OSI expression dialect (e.g. SNOWFLAKE); "
+ "ANSI_SQL is always the fallback")
+
+ imp = sub.add_parser("import", help="Omni model directory -> OSI semantic model YAML")
+ imp.add_argument("-i", "--input", required=True, help="Omni model directory")
+ imp.add_argument("-o", "--output", help="output OSI YAML file (default: stdout)")
+ imp.add_argument("--name", help="OSI model name (default: the mapped topic's name)")
+ imp.add_argument("--topic",
+ help="topic whose description/AI context map onto the OSI model "
+ "(default: the sole topic, if there is exactly one)")
+ return parser
+
+
+def _read_model_dir(path):
+ """Collect every YAML file under an Omni model directory as {relative path:
+ text}. Hidden files and non-YAML extensions (except the canonical
+ extensionless `model`/`relationships`/`*.view`/`*.topic` names) are skipped."""
+ if not os.path.isdir(path):
+ raise ConversionError(f"'{path}' is not a directory")
+ files = {}
+ for dirpath, dirnames, filenames in os.walk(path):
+ dirnames[:] = [d for d in sorted(dirnames) if not d.startswith(".")]
+ for fname in sorted(filenames):
+ if fname.startswith("."):
+ continue
+ rel = os.path.relpath(os.path.join(dirpath, fname), path)
+ rel = rel.replace(os.sep, "/")
+ base = fname.lower()
+ if not (base.endswith((".yaml", ".yml", ".view", ".topic"))
+ or base in ("model", "relationships")):
+ continue
+ with open(os.path.join(dirpath, fname)) as fh:
+ files[rel] = fh.read()
+ return files
+
+
+def main(argv=None):
+ args = _build_parser().parse_args(argv)
+ try:
+ if args.command == "export":
+ with open(args.input) as fh:
+ osi_yaml = fh.read()
+ files = convert_osi_to_omni(osi_yaml, base_view=args.base_view,
+ dialect=args.dialect)
+ for rel, text in files.items():
+ dest = os.path.join(args.output, *rel.split("/"))
+ os.makedirs(os.path.dirname(dest) or ".", exist_ok=True)
+ with open(dest, "w") as fh:
+ fh.write(text)
+ print(f"Wrote {len(files)} file(s) to {args.output}", file=sys.stderr)
+ else:
+ files = _read_model_dir(args.input)
+ out = convert_omni_to_osi(files, model_name=args.name, topic=args.topic)
+ if args.output:
+ with open(args.output, "w") as fh:
+ fh.write(out)
+ else:
+ sys.stdout.write(out)
+ except (ConversionError, OSError) as e:
+ print(f"Error: {e}", file=sys.stderr)
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/converters/omni/src/osi_omni/omni_to_osi.py b/converters/omni/src/osi_omni/omni_to_osi.py
new file mode 100644
index 0000000..b627509
--- /dev/null
+++ b/converters/omni/src/osi_omni/omni_to_osi.py
@@ -0,0 +1,701 @@
+"""Convert Omni semantic model files to an OSI semantic model.
+
+Pure offline conversion. Accepts the Omni model-directory layout as a mapping of
+{relative filename: YAML string} -- `model.yaml`, `relationships.yaml`,
+`views/*.view.yaml`, `topics/*.topic.yaml`. Omni features OSI has no native field
+for (formats, timeframes, hidden/tags, topic curation, the model file itself,
+query views, extends-only views, non-equi joins, ...) are preserved in
+`custom_extensions[OMNI]` so that converting back reproduces the original files.
+See README.md.
+
+Usage (CLI):
+ osi-omni import -i omni_model/ [-o model.yaml] [--name NAME] [--topic TOPIC]
+"""
+
+import re
+import warnings
+
+from ._common import (
+ ConversionError,
+ DIALECT_ANSI,
+ MODEL_FILE,
+ OSI_VERSION,
+ REL_MANY_TO_ONE,
+ REL_ONE_TO_MANY,
+ RELATIONSHIPS_FILE,
+ TOPIC_DIR,
+ dump_yaml,
+ has_timeframe_ref,
+ is_simple_identifier,
+ join_source,
+ load_yaml,
+ omni_sql_to_osi,
+ require_str,
+ view_file,
+ write_stash,
+)
+
+
+def _warn(scope, msg):
+ warnings.warn(f"[{scope}] {msg}")
+
+
+_VIEW_FILE_RE = re.compile(r"(?:^|/)([^/]+)\.view(?:\.ya?ml)?$")
+_QUERY_VIEW_FILE_RE = re.compile(r"(?:^|/)([^/]+)\.query\.view(?:\.ya?ml)?$")
+_TOPIC_FILE_RE = re.compile(r"(?:^|/)([^/]+)\.topic(?:\.ya?ml)?$")
+_MODEL_FILE_RE = re.compile(r"(?:^|/)model(?:\.ya?ml)?$")
+_RELS_FILE_RE = re.compile(r"(?:^|/)relationships(?:\.ya?ml)?$")
+
+# Omni's IDE/API writes each view file with a header naming the identifier the
+# rest of the model uses for it -- which is schema-qualified (`schema__table`)
+# when the view is outside the connection's default schema, and so differs from
+# the file's basename. That header is authoritative; the basename is the
+# fallback for hand-laid-out directories (including this converter's exports).
+_REF_COMMENT_RE = re.compile(r"^#\s*Reference this view as\s+([A-Za-z_]\w*)\s*$")
+
+
+def _canonical_view_name(text, basename):
+ for line in text.splitlines()[:5]:
+ m = _REF_COMMENT_RE.match(line)
+ if m:
+ return m.group(1)
+ if line.strip() and not line.lstrip().startswith("#"):
+ break
+ return basename
+
+# View-file keys the converter maps natively; everything else is stashed
+# verbatim in the dataset's `view_extras` (and restored on export).
+_VIEW_NATIVE_KEYS = {"schema", "catalog", "table_name", "sql", "description",
+ "ai_context", "dimensions", "measures",
+ "custom_compound_primary_key_sql"}
+
+# Dimension keys mapped natively; the rest stash flat on the field.
+_DIM_NATIVE_KEYS = {"sql", "label", "description", "synonyms", "ai_context",
+ "primary_key"}
+
+# Measure keys the OSI metric represents natively (given a reconstructible
+# aggregate); any other key forces the full-measure stash.
+_MEASURE_NATIVE_KEYS = {"sql", "aggregate_type", "description", "synonyms",
+ "ai_context"}
+
+# aggregate_type values whose OSI expression the exporter can rebuild exactly.
+_SIMPLE_AGGS = {"sum": "SUM", "count": "COUNT", "average": "AVG", "min": "MIN",
+ "max": "MAX", "median": "MEDIAN", "count_distinct": None}
+
+# Best-effort ANSI renderings for Omni-only aggregate types. The original
+# measure is stashed verbatim, so the Omni -> OSI -> Omni trip stays lossless;
+# the expression is what other OSI consumers see.
+_EXOTIC_AGGS = {"percentile", "list", "sum_distinct_on", "average_distinct_on",
+ "median_distinct_on", "percentile_distinct_on"}
+
+_ON_CLAUSE_RE = re.compile(
+ r"^\s*\$\{\s*([A-Za-z_][\w]*)\.([A-Za-z_][\w]*)\s*\}"
+ r"\s*=\s*"
+ r"\$\{\s*([A-Za-z_][\w]*)\.([A-Za-z_][\w]*)\s*\}\s*$"
+)
+
+
+def convert_omni_to_osi(files, model_name=None, topic=None):
+ """Convert Omni model files ({relative filename: YAML str}) to OSI YAML.
+
+ `model_name` overrides the OSI model name (default: the mapped topic's name,
+ else 'omni_model'). `topic` names the topic whose description/AI context map
+ onto the OSI model when the directory holds more than one topic.
+ """
+ if not isinstance(files, dict) or not files:
+ raise ConversionError("expected a non-empty mapping of {filename: YAML}")
+
+ views, topics = {}, {}
+ view_meta = {} # canonical name -> (file path, file basename)
+ topic_paths = {}
+ unmapped_views = set() # view names present as files but with no OSI dataset
+ model_yaml = None
+ rel_entries = []
+ extra_files = {}
+ for fname, text in files.items():
+ qv = _QUERY_VIEW_FILE_RE.search(fname)
+ if qv:
+ # Query views are backed by a saved query, not a table; OSI has no
+ # dataset form for them. Preserved verbatim, restored on export.
+ _warn(f"file '{fname}'", "query views have no OSI dataset form; "
+ "preserved in custom_extensions only")
+ unmapped_views.add(_canonical_view_name(text, qv.group(1)))
+ extra_files[fname] = text
+ continue
+ mv = _VIEW_FILE_RE.search(fname)
+ if mv:
+ vname = _canonical_view_name(text, mv.group(1))
+ parsed = load_yaml(text, fname) or {}
+ source = join_source(
+ dict(parsed, table_name=parsed.get("table_name", mv.group(1))))
+ if source is None:
+ # A view with no schema/sql of its own (an `extends`-only view)
+ # has no standalone OSI dataset form. Preserved verbatim.
+ _warn(f"view '{vname}'",
+ "no `schema`/`sql` source (an extends-only view?); "
+ "preserved in custom_extensions only")
+ unmapped_views.add(vname)
+ extra_files[fname] = text
+ continue
+ if vname in views:
+ raise ConversionError(
+ f"two view files resolve to view '{vname}' "
+ f"('{view_meta[vname][0]}' and '{fname}')")
+ views[vname] = parsed
+ view_meta[vname] = (fname, mv.group(1))
+ continue
+ mt = _TOPIC_FILE_RE.search(fname)
+ if mt:
+ topics[mt.group(1)] = load_yaml(text, fname) or {}
+ topic_paths[mt.group(1)] = fname
+ continue
+ if _MODEL_FILE_RE.search(fname):
+ model_yaml = load_yaml(text, fname)
+ continue
+ if _RELS_FILE_RE.search(fname):
+ parsed = load_yaml(text, fname) or []
+ if not isinstance(parsed, list):
+ raise ConversionError(
+ f"'{fname}' must be a top-level YAML list of joins")
+ rel_entries = parsed
+ continue
+ _warn(f"file '{fname}'", "unrecognized file; preserved in "
+ "custom_extensions only")
+ extra_files[fname] = text
+
+ if not views:
+ raise ConversionError(
+ "no convertible view files (*.view.yaml with a schema/sql source) "
+ "found; nothing to convert")
+
+ # The mapped topic supplies the OSI model's name/description/ai_context.
+ mapped_name = None
+ if topic is not None:
+ if topic not in topics:
+ raise ConversionError(
+ f"requested topic '{topic}' not found; topics present: "
+ f"{sorted(topics) or 'none'}")
+ mapped_name = topic
+ elif len(topics) == 1:
+ mapped_name = next(iter(topics))
+ elif len(topics) > 1:
+ _warn("model", f"{len(topics)} topics found and none chosen with --topic; "
+ f"topic metadata is preserved in custom_extensions only")
+
+ model = {"name": model_name or mapped_name or "omni_model"}
+
+ mapped_topic = topics.get(mapped_name, {})
+ if mapped_topic.get("description"):
+ model["description"] = mapped_topic["description"]
+ ai = {}
+ if mapped_topic.get("ai_context"):
+ ai["instructions"] = mapped_topic["ai_context"]
+ if ai:
+ model["ai_context"] = ai
+
+ datasets = []
+ for vname, view in views.items():
+ datasets.append(_convert_view(vname, view, view_meta[vname]))
+ model["datasets"] = datasets
+
+ # A join OSI cannot represent -- one touching a view with no OSI dataset (a
+ # query view, an extends-only view), or a non-equi/cross join -- is stashed
+ # verbatim with its position, so export rebuilds relationships.yaml in the
+ # original order. Malformed entries still raise.
+ relationships, extra_rels = [], []
+ rel_names = set()
+ for i, entry in enumerate(rel_entries):
+ endpoints = {entry.get("join_from_view"), entry.get("join_to_view")}
+ rel = None
+ if endpoints & unmapped_views:
+ _warn(f"relationship #{i + 1}",
+ "references a view with no OSI dataset form; preserved in "
+ "custom_extensions only")
+ else:
+ rel = _convert_relationship(entry, i, views)
+ if rel is None:
+ extra_rels.append({"index": i, "entry": entry})
+ continue
+ # OSI relationship names are unique per model; several (aliased) joins
+ # between one view pair generate the same `<from>_to_<to>` -- suffix
+ # the repeats. Export never reads the name, so this stays lossless.
+ base, n, k = rel["name"], rel["name"], 2
+ while n in rel_names:
+ n, k = f"{base}_{k}", k + 1
+ rel["name"] = n
+ rel_names.add(n)
+ relationships.append(rel)
+ if relationships:
+ model["relationships"] = relationships
+
+ base_view = mapped_topic.get("base_view")
+ metrics = _convert_measures(views, relationships, base_view)
+ if metrics:
+ model["metrics"] = metrics
+
+ # Model-level stash: the model file and topics verbatim (minus natively
+ # mapped topic properties), the mapped topic's identity, the topic's base
+ # view (so export re-roots the generated join tree identically), and any
+ # unconvertible files. `topics` is stashed even when empty so a lossless
+ # re-export does not invent a topic the original model never had.
+ stash = {"topics": {}}
+ if model_yaml is not None:
+ stash["model_file"] = model_yaml
+ for tname, tdict in topics.items():
+ tdict = dict(tdict)
+ if tname == mapped_name:
+ tdict.pop("description", None)
+ tdict.pop("ai_context", None)
+ stash["topics"][tname] = tdict
+ topic_files = {t: p for t, p in topic_paths.items()
+ if p != f"{TOPIC_DIR}/{t}.topic.yaml"}
+ if topic_files:
+ stash["topic_files"] = topic_files
+ if mapped_name is not None:
+ stash["mapped_topic"] = mapped_name
+ if base_view:
+ if base_view not in views:
+ _warn(f"topic '{mapped_name}'",
+ f"base_view '{base_view}' is not a view in this model")
+ else:
+ stash["base_view"] = base_view
+ if extra_rels:
+ stash["extra_relationships"] = extra_rels
+ if extra_files:
+ stash["extra_files"] = extra_files
+ write_stash(model, stash)
+
+ return dump_yaml({"version": OSI_VERSION, "semantic_model": [model]})
+
+
+def _convert_view(vname, view, meta):
+ scope = f"view '{vname}'"
+ fname, basename = meta
+ ds = {"name": vname}
+ stash = {}
+
+ # An implicit table_name is the *file's* name -- not the canonical view
+ # name, which is schema-qualified for a view outside the default schema.
+ source = join_source(dict(view, table_name=view.get("table_name", basename)))
+ ds["source"] = source
+ if str(view.get("table_name", "")) == basename:
+ # Explicit-but-redundant table_name: the exporter would normalize it
+ # away, so remember it was spelled out.
+ stash["table_name"] = view["table_name"]
+
+ if view.get("description"):
+ ds["description"] = view["description"]
+ if view.get("ai_context"):
+ ds["ai_context"] = {"instructions": view["ai_context"]}
+
+ fields = []
+ pk_cols = []
+ # Omni writes compound-key entries as `${view.field}`/`${field}` references;
+ # resolve same-view references to plain field names (the original list is
+ # stashed whenever this normalization changes it).
+ raw_compound = view.get("custom_compound_primary_key_sql") or []
+ compound = []
+ for c in raw_compound:
+ translated, _ = omni_sql_to_osi(str(c), vname)
+ translated = translated.strip()
+ compound.append(translated if is_simple_identifier(translated) else str(c))
+ # Omni mustache templating ({{# field.filter }} ...) in a field's sql has
+ # no SQL (or OSI) form at all; such dimensions/measures are stashed whole
+ # and dropped from the OSI model. Popped here so the later measure pass
+ # sees only convertible measures.
+ for kind, key in (("dimensions", "extra_dimensions"),
+ ("measures", "extra_measures")):
+ entries = view.get(kind) or {}
+ templated = {n: e for n, e in entries.items()
+ if "{{" in str((e or {}).get("sql", ""))}
+ if templated:
+ for n in templated:
+ _warn(f"{kind[:-1]} '{vname}.{n}'",
+ "sql uses Omni template syntax ('{{'), which has no "
+ "OSI form; preserved in custom_extensions only")
+ entries.pop(n)
+ stash[key] = templated
+
+ dims = view.get("dimensions") or {}
+ covered = set() # names the exporter can resolve back to a dimension
+ for dname, dim in dims.items():
+ dim = dim or {}
+ field, col = _convert_dimension(vname, dname, dim)
+ fields.append(field)
+ covered.update((dname, col))
+ if dim.get("primary_key"):
+ pk_cols.append(col)
+ if dname in compound:
+ compound = [col if c == dname else c for c in compound]
+ if fields:
+ ds["fields"] = fields
+
+ # Stash the original compound-key list whenever the exporter could not
+ # rebuild it from the OSI primary_key alone (a `${view.field}` reference
+ # that was normalized away, or an entry no dimension covers).
+ if raw_compound and (
+ compound != [str(c) for c in raw_compound]
+ or any(str(c) not in covered for c in compound)):
+ stash["custom_compound_primary_key_sql"] = list(raw_compound)
+
+ if compound:
+ unknown = [c for c in compound if not is_simple_identifier(str(c))]
+ if unknown:
+ _warn(scope, f"custom_compound_primary_key_sql entries {unknown} are not "
+ f"plain field names; using them as-is in primary_key")
+ ds["primary_key"] = [str(c) for c in compound]
+ if pk_cols:
+ _warn(scope, "both a primary_key dimension and "
+ "custom_compound_primary_key_sql found; using the compound key")
+ elif len(pk_cols) == 1:
+ ds["primary_key"] = pk_cols
+ elif len(pk_cols) > 1:
+ # Multiple primary_key dimensions form a composite key in Omni.
+ ds["primary_key"] = pk_cols
+
+ if fname != view_file(vname):
+ stash["file"] = fname
+ extras = {k: v for k, v in view.items() if k not in _VIEW_NATIVE_KEYS}
+ for key in ("description", "ai_context"):
+ # Present-but-empty metadata has no OSI slot; preserve it as an extra.
+ if key in view and not view[key]:
+ extras[key] = view[key]
+ if extras:
+ stash["view_extras"] = extras
+ write_stash(ds, stash)
+ return ds
+
+
+def _convert_dimension(vname, dname, dim):
+ """Build one OSI field from an Omni dimension. Returns (field, column) where
+ `column` is the underlying column used for key resolution (the translated
+ expression when it is a bare column, else the dimension name)."""
+ scope = f"dimension '{vname}.{dname}'"
+ stash = {}
+
+ sql = dim.get("sql")
+ if sql is None:
+ expr = dname # schema-layer default: the same-named physical column
+ else:
+ sql = str(sql)
+ expr, changed = omni_sql_to_osi(sql, vname)
+ if has_timeframe_ref(sql):
+ _warn(scope, "timeframe reference (${view.field[timeframe]}) has no OSI "
+ "form; flattened to the base field, original sql stashed")
+ if changed or sql.strip() == dname:
+ # Stashed when the OSI expression differs from the Omni sql, and
+ # also when the sql is an explicit same-named bare column -- which
+ # the exporter would otherwise normalize to the implicit
+ # schema-layer default (no `sql:` key).
+ stash["sql"] = sql
+
+ field = {
+ "name": dname,
+ "expression": {"dialects": [{"dialect": DIALECT_ANSI, "expression": expr}]},
+ }
+ if dim.get("label"):
+ field["label"] = dim["label"]
+ if dim.get("description"):
+ field["description"] = dim["description"]
+ ai = {}
+ if dim.get("ai_context"):
+ ai["instructions"] = dim["ai_context"]
+ if dim.get("synonyms"):
+ ai["synonyms"] = list(dim["synonyms"])
+ if ai:
+ field["ai_context"] = ai
+ if "timeframes" in dim:
+ field["dimension"] = {"is_time": True}
+ stash["timeframes"] = dim["timeframes"]
+
+ for key, value in dim.items():
+ if key == "timeframes":
+ continue
+ if key in _DIM_NATIVE_KEYS:
+ # A present-but-empty native value (description: '') has no OSI
+ # slot -- OSI omits empty metadata -- so it rides in the stash.
+ if not value and not isinstance(value, bool) and key != "sql":
+ stash[key] = value
+ continue
+ stash[key] = value
+
+ write_stash(field, stash)
+ column = expr.strip() if is_simple_identifier(expr) else dname
+ return field, column
+
+
+def _convert_relationship(entry, index, views):
+ what = f"relationship #{index + 1}"
+ from_view = require_str(entry, "join_from_view", what)
+ to_view = require_str(entry, "join_to_view", what)
+ for v in (from_view, to_view):
+ if v not in views:
+ raise ConversionError(f"{what}: view '{v}' has no view file")
+ on_sql = require_str(entry, "on_sql", what)
+
+ # Aliased joins reference the alias in on_sql; accept those names too.
+ aliases = {
+ entry.get("join_from_view_as") or from_view: from_view,
+ entry.get("join_to_view_as") or to_view: to_view,
+ from_view: from_view,
+ to_view: to_view,
+ }
+
+ from_cols, to_cols = [], []
+ for clause in re.split(r"\s+AND\s+", on_sql, flags=re.IGNORECASE):
+ m = _ON_CLAUSE_RE.match(clause)
+ if not m:
+ # A valid Omni join OSI cannot express (a range/non-equi or cross
+ # join). The caller stashes the entry verbatim.
+ _warn(what, f"('{from_view}' -> '{to_view}'): on_sql clause "
+ f"'{clause.strip()}' is not an equi-join of two "
+ f"${{view.field}} references, so it has no OSI "
+ f"relationship form; preserved in custom_extensions only")
+ return None
+ la, lf, ra, rf = m.groups()
+ if aliases.get(la) == from_view and aliases.get(ra) == to_view:
+ from_cols.append(_field_column(views[from_view], lf))
+ to_cols.append(_field_column(views[to_view], rf))
+ elif aliases.get(la) == to_view and aliases.get(ra) == from_view:
+ from_cols.append(_field_column(views[from_view], rf))
+ to_cols.append(_field_column(views[to_view], lf))
+ else:
+ _warn(what, f"on_sql clause '{clause.strip()}' references views other "
+ f"than '{from_view}'/'{to_view}' (or their aliases); "
+ f"preserved in custom_extensions only")
+ return None
+
+ # The declared type/join_type are stashed verbatim whenever present -- even
+ # when they restate an Omni default -- so export reproduces the exact file.
+ stash = {}
+ # Export rebuilds on_sql from the OSI columns in canonical form; when that
+ # would not reproduce the original (an alias reference, reversed clause
+ # sides, `and` casing, spacing, a field-to-column translation), the
+ # original rides in the stash instead.
+ rebuilt = " AND ".join(
+ "${" + f"{from_view}.{fc}" + "} = ${" + f"{to_view}.{tc}" + "}"
+ for fc, tc in zip(from_cols, to_cols))
+ if rebuilt != on_sql:
+ stash["on_sql"] = on_sql
+ rel_type = str(entry.get("relationship_type") or REL_MANY_TO_ONE)
+ if "relationship_type" in entry:
+ stash["relationship_type"] = rel_type
+ if rel_type == REL_ONE_TO_MANY:
+ # OSI `from` is always the many side; flip the orientation (the stashed
+ # type tells export to flip back to the original one_to_many join).
+ from_view, to_view = to_view, from_view
+ from_cols, to_cols = to_cols, from_cols
+ elif rel_type == "many_to_many":
+ # one_to_one / many_to_many / assumed_many_to_one keep the declared
+ # orientation.
+ _warn(what, "many_to_many has no OSI orientation (OSI `to` is the one "
+ "side); orientation kept as declared, type preserved in "
+ "custom_extensions")
+
+ rel = {"name": f"{from_view}_to_{to_view}", "from": from_view, "to": to_view,
+ "from_columns": from_cols, "to_columns": to_cols}
+
+ if entry.get("join_type"):
+ stash["join_type"] = entry["join_type"]
+ if entry.get("where_sql"):
+ stash["where_sql"] = entry["where_sql"]
+ _warn(what, "where_sql (join filter) has no OSI form; preserved in "
+ "custom_extensions only")
+ for key in ("reversible", "id", "join_from_view_as", "join_from_view_as_label",
+ "join_to_view_as", "join_to_view_as_label"):
+ if key in entry:
+ stash[key] = entry[key]
+ write_stash(rel, stash)
+ return rel
+
+
+def _field_column(view, fname):
+ """Resolve a ${view.field} reference to the field's underlying column when the
+ dimension is a bare column, else keep the field name."""
+ dim = (view.get("dimensions") or {}).get(fname) or {}
+ sql = dim.get("sql")
+ if sql is None:
+ return fname
+ return sql.strip() if is_simple_identifier(str(sql)) else fname
+
+
+def _convert_measures(views, relationships, base_view):
+ """Turn every view's measures into OSI model-level metrics.
+
+ A metric name is the measure name when globally unique, else
+ `<view>__<measure>` (the original name is stashed either way when it
+ matters). The full original measure is stashed whenever the OSI expression
+ alone cannot reconstruct it exactly.
+ """
+ # The exporter re-derives measure placement from the expression (a
+ # view-qualified aggregate lands on that view, everything else on the base
+ # view). Compute that default here so placement is only stashed when needed.
+ effective_base = base_view or _fk_sink(views, relationships)
+
+ counts = {}
+ for vname, view in views.items():
+ for mname in (view.get("measures") or {}):
+ counts[mname] = counts.get(mname, 0) + 1
+
+ metrics = []
+ seen = set()
+ for vname, view in views.items():
+ for mname, measure in (view.get("measures") or {}).items():
+ measure = measure or {}
+ metric_name = mname if counts[mname] == 1 else f"{vname}__{mname}"
+ if metric_name in seen:
+ raise ConversionError(
+ f"metric name '{metric_name}' derived twice; rename the "
+ f"colliding measures in Omni")
+ seen.add(metric_name)
+ metrics.append(
+ _convert_measure(vname, mname, metric_name, measure, effective_base))
+ return metrics
+
+
+def _convert_measure(vname, mname, metric_name, measure, effective_base):
+ scope = f"measure '{vname}.{mname}'"
+ stash = {}
+
+ agg = measure.get("aggregate_type")
+ sql = measure.get("sql")
+ expr = None
+ # Reconstructible = the exporter can rebuild this measure from the OSI
+ # expression alone: only natively-mapped keys, and a sql that is either free
+ # of `${...}` references or a lone same-view field reference (which the
+ # exporter re-derives from the view's dimension names). Anything else keeps
+ # the original measure in the stash.
+ reconstructible = set(measure) <= _MEASURE_NATIVE_KEYS and (
+ sql is None
+ or "${" not in str(sql)
+ or re.fullmatch(r"\s*\$\{\s*[A-Za-z_]\w*\s*\}\s*", str(sql)) is not None
+ )
+
+ if agg is None and sql is not None:
+ # A raw-SQL measure (the aggregate is written out in the sql). OSI
+ # metrics are model-level, so view qualifiers are kept, not stripped
+ # (own_view=None below leaves `${view.field}` as `view.field`).
+ # Always stashed: the exporter would otherwise re-parse a lone
+ # aggregate call into a structured measure, changing the file.
+ expr, _ = omni_sql_to_osi(str(sql), own_view=None)
+ reconstructible = False
+ elif agg in _SIMPLE_AGGS:
+ if agg == "count" and sql is None:
+ expr = "COUNT(*)"
+ elif sql is None:
+ raise ConversionError(f"{scope}: aggregate_type '{agg}' requires sql")
+ else:
+ inner = _measure_operand(vname, str(sql))
+ if agg == "count_distinct":
+ expr = f"COUNT(DISTINCT {inner})"
+ else:
+ expr = f"{_SIMPLE_AGGS[agg]}({inner})"
+ elif agg in _EXOTIC_AGGS:
+ # Best-effort ANSI so OSI consumers still see a metric; the verbatim
+ # measure is stashed, so re-export is exact.
+ inner = _measure_operand(vname, str(sql or ""))
+ expr = _exotic_expr(agg, inner, measure)
+ _warn(scope, f"aggregate_type '{agg}' has no exact ANSI form; emitted a "
+ f"best-effort expression, original measure preserved in "
+ f"custom_extensions")
+ reconstructible = False
+ else:
+ raise ConversionError(
+ f"{scope}: unknown aggregate_type '{agg}'")
+
+ if measure.get("filters"):
+ _warn(scope, "measure filters have no OSI form; expression emitted "
+ "unfiltered, original measure preserved in custom_extensions")
+ reconstructible = False
+
+ # Placement/name recovery: only stashed when the exporter would otherwise
+ # derive a different view or name.
+ derived_target = _derived_placement(expr, effective_base)
+ if not reconstructible:
+ stashed_measure = {
+ k: v for k, v in measure.items()
+ if k not in ("description", "synonyms", "ai_context")
+ }
+ stash["measure"] = stashed_measure
+ stash["view"] = vname
+ if metric_name != mname:
+ stash["name"] = mname
+ else:
+ if derived_target != vname:
+ stash["view"] = vname
+ if metric_name != mname:
+ stash["name"] = mname
+
+ metric = {
+ "name": metric_name,
+ "expression": {"dialects": [{"dialect": DIALECT_ANSI, "expression": expr}]},
+ }
+ if measure.get("description"):
+ metric["description"] = measure["description"]
+ ai = {}
+ if measure.get("ai_context"):
+ ai["instructions"] = measure["ai_context"]
+ if measure.get("synonyms"):
+ ai["synonyms"] = list(measure["synonyms"])
+ if ai:
+ metric["ai_context"] = ai
+ write_stash(metric, stash)
+ return metric
+
+
+def _measure_operand(vname, sql):
+ """Translate a measure's operand sql to an OSI reference: a same-view field
+ or bare column becomes `view.name` (the qualified form OSI metrics use);
+ anything else keeps its view qualifiers and is left as-is."""
+ inner, _ = omni_sql_to_osi(sql, own_view=None)
+ inner = inner.strip()
+ if is_simple_identifier(inner):
+ return f"{vname}.{inner}"
+ return inner
+
+
+def _derived_placement(expr, effective_base):
+ """Mirror the exporter's placement rule: a metric lands on the single view
+ its expression references, else on the base view."""
+ refs = set(re.findall(r"(?<![\w.$])([A-Za-z_][\w]*)\.[A-Za-z_][\w]*(?![\w.])",
+ expr or ""))
+ if len(refs) == 1:
+ return next(iter(refs))
+ return effective_base
+
+
+def _fk_sink(views, relationships):
+ """The dataset that is never a relationship `to` -- the exporter's default
+ base view. None when it is ambiguous (the exporter would demand a hint)."""
+ if len(views) == 1:
+ return next(iter(views))
+ incoming = {name: 0 for name in views}
+ for rel in relationships or []:
+ if rel["to"] in incoming:
+ incoming[rel["to"]] += 1
+ roots = [n for n in views if incoming[n] == 0]
+ return roots[0] if len(roots) == 1 else None
+
+
+def _exotic_expr(agg, inner, measure):
+ if agg == "percentile":
+ p = measure.get("percentile", 50)
+ try:
+ frac = float(p) / 100.0
+ except (TypeError, ValueError):
+ frac = 0.5
+ return f"PERCENTILE_CONT({frac}) WITHIN GROUP (ORDER BY {inner})"
+ if agg == "list":
+ return f"LISTAGG({inner}, ', ')"
+ # *_distinct_on: the dedup key has no ANSI slot; approximate with the plain
+ # aggregate over the operand.
+ base = {"sum_distinct_on": "SUM", "average_distinct_on": "AVG",
+ "median_distinct_on": "MEDIAN"}.get(agg)
+ if base:
+ return f"{base}({inner})"
+ p = measure.get("percentile", 50)
+ try:
+ frac = float(p) / 100.0
+ except (TypeError, ValueError):
+ frac = 0.5
+ return f"PERCENTILE_CONT({frac}) WITHIN GROUP (ORDER BY {inner})"
diff --git a/converters/omni/src/osi_omni/osi_to_omni.py b/converters/omni/src/osi_omni/osi_to_omni.py
new file mode 100644
index 0000000..bcbb351
--- /dev/null
+++ b/converters/omni/src/osi_omni/osi_to_omni.py
@@ -0,0 +1,609 @@
+"""Convert an OSI semantic model to Omni semantic model files.
+
+Pure offline conversion -- no Omni connection required. Produces the Omni
+model-directory layout: one `views/<name>.view.yaml` per dataset, a
+`relationships.yaml` join list, one generated `topics/<name>.topic.yaml`
+(base view chosen like a fact table), and -- when a prior import stashed one --
+the original `model.yaml`. See README.md for the capability summary.
+
+Usage (CLI):
+ osi-omni export -i model.yaml -o omni_model/ [--base-view orders] [--dialect SNOWFLAKE]
+"""
+
+import re
+import warnings
+
+from ._common import (
+ ConversionError,
+ DEFAULT_TIMEFRAMES,
+ MODEL_FILE,
+ OSI_VERSION,
+ REL_MANY_TO_ONE,
+ REL_ONE_TO_MANY,
+ RELATIONSHIPS_FILE,
+ dump_yaml,
+ foreign_vendor_extensions,
+ instructions_of,
+ is_simple_identifier,
+ load_yaml,
+ osi_expr_refs_to_omni,
+ parse_source,
+ pick_expression,
+ read_stash,
+ require_str,
+ sanitize_name,
+ synonyms_of,
+ topic_file,
+ view_file,
+ write_stash, # noqa: F401 (re-exported for symmetry in tests)
+)
+
+
+def _warn(scope, msg):
+ warnings.warn(f"[{scope}] {msg}")
+
+
+# Dimension-level stash keys restored verbatim onto the exported dimension.
+# `sql` is handled separately (it replaces the derived expression).
+_DIM_STASH_PASSTHROUGH_EXCLUDE = {"sql"}
+
+# One OSI metric aggregate call maps to a structured Omni measure.
+_AGG_TO_OMNI = {
+ "SUM": "sum",
+ "COUNT": "count",
+ "AVG": "average",
+ "MIN": "min",
+ "MAX": "max",
+ "MEDIAN": "median",
+}
+
+_AGG_CALL_RE = re.compile(
+ r"^\s*(SUM|COUNT|AVG|MIN|MAX|MEDIAN)\s*\((.*)\)\s*$",
+ re.IGNORECASE | re.DOTALL,
+)
+
+# `view.column` -- a dotted reference an OSI metric uses to point into a dataset.
+_DOTTED_REF_RE = re.compile(
+ r"(?<![\w.$])([A-Za-z_][A-Za-z0-9_]*)\.([A-Za-z_][A-Za-z0-9_]*)(?![\w.])"
+)
+
+
+def convert_osi_to_omni(osi_yaml_str, base_view=None, dialect=None):
+ """Parse OSI YAML and return Omni model files as {relative filename: YAML str}.
+
+ `base_view` names the dataset the generated topic is rooted at. When omitted,
+ it is the dataset that is never a relationship `to` (the FK sink of a plain
+ many-to-one star). `dialect` prepends a warehouse dialect (e.g. SNOWFLAKE) to
+ the expression preference order; ANSI_SQL is always the fallback.
+ """
+ root = load_yaml(osi_yaml_str, "OSI model")
+ if not isinstance(root, dict):
+ raise ConversionError("Invalid OSI YAML: expected a mapping at the root")
+
+ version = str(root.get("version", ""))
+ if version != OSI_VERSION:
+ raise ConversionError(
+ f"Unsupported OSI version '{version}'. Supported: {OSI_VERSION}"
+ )
+
+ models = root.get("semantic_model")
+ if not isinstance(models, list) or not models:
+ raise ConversionError("'semantic_model' must be a non-empty list")
+ if len(models) > 1:
+ _warn("model", "multiple semantic models found; converting only the first")
+
+ return _convert_model(models[0], base_view, dialect)
+
+
+def _convert_model(model, explicit_base_view, dialect):
+ name = model.get("name", "<unnamed>")
+ dataset_list = model.get("datasets", []) or []
+ if not dataset_list:
+ raise ConversionError(f"Model '{name}' has no datasets")
+
+ # Dataset -> Omni view names. Sanitization collisions (and case-insensitive
+ # duplicates, which sanitize identically) fail loudly rather than merging.
+ view_names = {}
+ taken = set()
+ for d in dataset_list:
+ ds_name = require_str(d, "name", f"Model '{name}': dataset")
+ view_names[ds_name] = sanitize_name(ds_name, f"Model '{name}': dataset", taken)
+ taken.add(view_names[ds_name].lower())
+ datasets = {d["name"]: d for d in dataset_list}
+ relationships = model.get("relationships", []) or []
+ for rel in relationships:
+ scope = f"Model '{name}': relationship '{rel.get('name', '<unnamed>')}'"
+ if (require_str(rel, "from", scope) not in datasets
+ or require_str(rel, "to", scope) not in datasets):
+ raise ConversionError(f"{scope} references an unknown dataset")
+
+ model_stash = read_stash(model)
+
+ files = {}
+
+ # model.yaml: only a stashed original (a fresh model needs no model file;
+ # model-wide Omni settings have no OSI source to generate from).
+ if model_stash.get("model_file") is not None:
+ files[MODEL_FILE] = dump_yaml(model_stash["model_file"])
+
+ # Views (and the per-view dimension name maps the other stages need). A
+ # stashed original file path (e.g. `DELIGHTED/response.view`) wins over the
+ # canonical `views/<name>.view.yaml` layout.
+ dims_by_view = {}
+ view_paths = {}
+ for ds_name, ds in datasets.items():
+ vname = view_names[ds_name]
+ view, dim_names = _convert_dataset(ds, vname, view_names, dialect)
+ view_paths[vname] = read_stash(ds).get("file") or view_file(vname)
+ files[view_paths[vname]] = dump_yaml(view)
+ dims_by_view[vname] = dim_names
+
+ # Relationships: converted OSI relationships in order, with any joins a
+ # prior import could not map (they touch a query/extends view) reinserted
+ # verbatim at their original positions.
+ imported = "topics" in model_stash
+ rel_entries = [
+ _convert_relationship(rel, view_names, imported) for rel in relationships
+ ]
+ for item in sorted(model_stash.get("extra_relationships") or [],
+ key=lambda x: x.get("index", 0)):
+ rel_entries.insert(min(item.get("index", 0), len(rel_entries)),
+ item["entry"])
+ if rel_entries:
+ files[RELATIONSHIPS_FILE] = dump_yaml(rel_entries)
+
+ # Base view: explicit flag > stashed original > FK-sink heuristic. Resolved
+ # lazily -- a model whose topics are stashed and whose metrics all carry (or
+ # derive) their own placement never needs one, so the multi-root error only
+ # fires when a base view is genuinely required.
+ base_hint = explicit_base_view or model_stash.get("base_view")
+ base_cache = []
+
+ def resolve_base():
+ if not base_cache:
+ base_cache.append(
+ view_names[_pick_base_view(name, datasets, relationships, base_hint)])
+ return base_cache[0]
+
+ # Measures (from OSI metrics) attach to views.
+ measures_by_view = {}
+ for metric in model.get("metrics", []) or []:
+ placed = _convert_metric(metric, resolve_base, view_names, dims_by_view,
+ dialect)
+ if placed is None:
+ continue
+ target_view, mname, measure = placed
+ target = measures_by_view.setdefault(target_view, {})
+ if mname.lower() in {m.lower() for m in target}:
+ raise ConversionError(
+ f"Model '{name}': two metrics map to measure '{mname}' on view "
+ f"'{target_view}'; rename one in the OSI model.")
+ target[mname] = measure
+ for vname, measures in measures_by_view.items():
+ view = load_yaml(files[view_paths[vname]], f"view '{vname}'") or {}
+ clashes = set(measures) & set(view.get("dimensions") or {})
+ for c in sorted(clashes):
+ _warn(f"measure '{c}'",
+ f"name collides with a dimension on view '{vname}'; Omni requires "
+ f"unique field names per view -- rename before use")
+ existing = view.setdefault("measures", {})
+ existing.update(measures)
+ files[view_paths[vname]] = dump_yaml(view)
+
+ # Topics: stashed originals restore verbatim (with natively-mapped properties
+ # re-injected on the mapped topic). The `topics` stash key being *present* --
+ # even empty -- means the original Omni model's topic set is known, so a
+ # fresh topic is only generated for hand-authored OSI (no stash).
+ if "topics" in model_stash:
+ mapped = model_stash.get("mapped_topic")
+ topic_paths = model_stash.get("topic_files") or {}
+ for tname, topic in (model_stash["topics"] or {}).items():
+ topic = dict(topic)
+ if tname == mapped:
+ if model.get("description"):
+ topic["description"] = model["description"]
+ instructions = instructions_of(model.get("ai_context"))
+ if instructions:
+ topic["ai_context"] = instructions
+ files[topic_paths.get(tname) or topic_file(tname)] = dump_yaml(topic)
+ else:
+ tname = sanitize_name(name, f"Model '{name}'", set())
+ files[topic_file(tname)] = dump_yaml(
+ _build_topic(model, resolve_base(), view_names, relationships))
+
+ # Files a prior import could not convert (query views, unrecognized files)
+ # restore verbatim.
+ for fname, text in (model_stash.get("extra_files") or {}).items():
+ files[fname] = text
+
+ _warn_dropped_model(model)
+ return files
+
+
+def _convert_dataset(ds, vname, view_names, dialect):
+ """Build one Omni view dict from an OSI dataset. Returns (view, dims) where
+ dims = {"cols": {column: dimension name}, "names": set of dimension names}
+ (used to resolve primary keys and metric references)."""
+ ds_name = ds["name"]
+ scope = f"dataset '{ds_name}'"
+ stash = read_stash(ds)
+
+ view = {}
+ parsed = parse_source(ds.get("source"), ds_name)
+ if parsed[0] == "sql":
+ view["sql"] = parsed[1]
+ else:
+ _, catalog, schema, table = parsed
+ if catalog:
+ view["catalog"] = catalog
+ view["schema"] = schema
+ # table_name defaults to the *file's* name -- the basename of the
+ # stashed original path when there is one (a schema-folder layout names
+ # the view `schema__table` but the file `SCHEMA/table.view`), else the
+ # view name. Emit only when it differs.
+ default_table = vname
+ if stash.get("file"):
+ base = stash["file"].rsplit("/", 1)[-1]
+ default_table = re.sub(r"\.view(\.ya?ml)?$", "", base)
+ if "table_name" in stash: # was spelled out even though redundant
+ view["table_name"] = stash["table_name"]
+ elif table != default_table:
+ view["table_name"] = table
+ if ds.get("description"):
+ view["description"] = ds["description"]
+ instructions = instructions_of(ds.get("ai_context"))
+ if instructions:
+ view["ai_context"] = instructions
+ if synonyms_of(ds.get("ai_context")):
+ _warn(scope, "dataset ai_context synonyms have no Omni view-level home; dropped")
+
+ # Restore stashed Omni view extras (label, hidden, tags, filters:, ...)
+ # verbatim. Keys the converter derives itself are not stashed on import.
+ for key, value in (stash.get("view_extras") or {}).items():
+ view[key] = value
+
+ dimensions = {}
+ col_to_dim = {}
+ taken = set()
+ for field in ds.get("fields", []) or []:
+ fname = require_str(field, "name", f"{scope}: field")
+ fscope = f"field '{fname}'"
+ dname = sanitize_name(fname, f"{scope}: field", taken)
+ taken.add(dname.lower())
+ expr = pick_expression(field.get("expression"), dialect)
+ if expr is None:
+ _warn(fscope, "no usable ANSI_SQL (or preferred-dialect) expression; "
+ "dropping field")
+ continue
+ dim = _convert_field(field, dname, expr, fscope)
+ dimensions[dname] = dim
+ if is_simple_identifier(expr):
+ col_to_dim[expr.strip()] = dname
+
+ # primary_key: mark the matching dimension (creating a hidden one for a key
+ # column no field covers); a composite key becomes the view-level
+ # custom_compound_primary_key_sql list of field names -- restored verbatim
+ # when a prior import stashed the original (`${view.field}`-form) list.
+ pk = ds.get("primary_key") or []
+ if stash.get("custom_compound_primary_key_sql"):
+ view["custom_compound_primary_key_sql"] = \
+ stash["custom_compound_primary_key_sql"]
+ elif pk:
+ pk_dims = []
+ for col in pk:
+ # A key entry is either a raw column some dimension covers, or
+ # already a dimension name (import resolves a computed dimension's
+ # key to its field name).
+ dname = col_to_dim.get(col) or (col if col in dimensions else None)
+ if dname is None:
+ dname = sanitize_name(col, f"{scope}: primary key column", taken)
+ taken.add(dname.lower())
+ dim = {"hidden": True}
+ if dname != col:
+ dim["sql"] = col
+ dimensions[dname] = dim
+ col_to_dim[col] = dname
+ pk_dims.append(dname)
+ if len(pk_dims) == 1:
+ dimensions[pk_dims[0]]["primary_key"] = True
+ else:
+ view["custom_compound_primary_key_sql"] = pk_dims
+
+ if ds.get("unique_keys"):
+ # A unique key that merely restates the primary key is redundant, not lost.
+ extra = [k for k in ds["unique_keys"] if list(k) != list(pk)]
+ if extra:
+ _warn(scope, "unique_keys have no Omni home; dropped")
+
+ # Fields a prior import could not convert (Omni template syntax in sql)
+ # restore verbatim alongside the mapped ones.
+ dimensions.update(stash.get("extra_dimensions") or {})
+ if stash.get("extra_measures"):
+ view["measures"] = dict(stash["extra_measures"])
+ if dimensions:
+ view["dimensions"] = dimensions
+ if foreign_vendor_extensions(ds):
+ _warn(scope, "foreign-vendor custom_extensions dropped")
+ return view, {"cols": col_to_dim, "names": set(dimensions)}
+
+
+def _convert_field(field, dname, expr, fscope):
+ stash = read_stash(field)
+ dim = {}
+
+ # A stashed original Omni `sql` (import rewrote its ${...} refs) wins, so
+ # Omni -> OSI -> Omni restores the exact expression.
+ if "sql" in stash:
+ dim["sql"] = stash["sql"]
+ elif is_simple_identifier(expr):
+ if expr.strip() != dname:
+ dim["sql"] = expr.strip()
+ # else: the dimension inherits the same-named schema column -- no sql key.
+ else:
+ # Raw SQL over the view's own columns is legal in Omni sql. References to
+ # other views would need ${view.field} form; OSI field expressions are
+ # dataset-scoped, so this is emitted as-is.
+ dim["sql"] = expr
+
+ if field.get("label"):
+ dim["label"] = field["label"]
+ if field.get("description"):
+ dim["description"] = field["description"]
+ syns = synonyms_of(field.get("ai_context"))
+ if syns:
+ dim["synonyms"] = syns
+ instructions = instructions_of(field.get("ai_context"))
+ if instructions:
+ dim["ai_context"] = instructions
+
+ if (field.get("dimension") or {}).get("is_time"):
+ # The stashed list wins even when empty (`timeframes: []` is a real
+ # Omni value); the default list is only for hand-authored OSI.
+ dim["timeframes"] = (stash["timeframes"] if "timeframes" in stash
+ else list(DEFAULT_TIMEFRAMES))
+ elif "timeframes" in stash:
+ dim["timeframes"] = stash["timeframes"]
+
+ for key, value in stash.items():
+ if key in ("sql", "timeframes") or key in dim:
+ continue
+ dim[key] = value
+
+ if foreign_vendor_extensions(field):
+ _warn(fscope, "foreign-vendor custom_extensions dropped")
+ return dim
+
+
+def _convert_relationship(rel, view_names, imported=False):
+ rname = rel.get("name", "<unnamed>")
+ scope = f"relationship '{rname}'"
+ from_cols = rel.get("from_columns") or []
+ to_cols = rel.get("to_columns") or []
+ if not isinstance(from_cols, list) or not isinstance(to_cols, list) \
+ or not from_cols or not to_cols:
+ raise ConversionError(
+ f"Relationship '{rname}': from_columns and to_columns are required lists")
+ if len(from_cols) != len(to_cols):
+ raise ConversionError(
+ f"Relationship '{rname}': from_columns ({len(from_cols)}) and "
+ f"to_columns ({len(to_cols)}) must have the same length")
+
+ stash = read_stash(rel)
+ from_view, to_view = view_names[rel["from"]], view_names[rel["to"]]
+ if stash.get("relationship_type") == REL_ONE_TO_MANY:
+ # The import flipped a one_to_many join so the OSI `from` is the many
+ # side; flip back to the originally declared orientation.
+ from_view, to_view = to_view, from_view
+ from_cols, to_cols = to_cols, from_cols
+ entry = {"join_from_view": from_view, "join_to_view": to_view}
+ # Aliased joins (join_*_view_as) are Omni-only; restore before on_sql so the
+ # stashed on_sql (which may reference the alias) stays consistent.
+ for key in ("join_from_view_as", "join_from_view_as_label",
+ "join_to_view_as", "join_to_view_as_label"):
+ if key in stash:
+ entry[key] = stash[key]
+
+ if "on_sql" in stash:
+ entry["on_sql"] = stash["on_sql"]
+ else:
+ entry["on_sql"] = " AND ".join(
+ "${" + f"{from_view}.{fc}" + "} = ${" + f"{to_view}.{tc}" + "}"
+ for fc, tc in zip(from_cols, to_cols)
+ )
+ # OSI orientation is many(from) -> one(to). The stash holds the declared
+ # type/join_type verbatim (even an explicit Omni default). An imported join
+ # with no stashed key genuinely omitted it; fresh OSI states many_to_one.
+ if "relationship_type" in stash:
+ entry["relationship_type"] = stash["relationship_type"]
+ elif not imported:
+ entry["relationship_type"] = REL_MANY_TO_ONE
+ if stash.get("join_type"):
+ entry["join_type"] = stash["join_type"]
+ for key in ("reversible", "where_sql", "id"):
+ if key in stash:
+ entry[key] = stash[key]
+
+ if rel.get("ai_context"):
+ _warn(scope, "relationship ai_context has no Omni home; dropped")
+ if foreign_vendor_extensions(rel):
+ _warn(scope, "foreign-vendor custom_extensions dropped")
+ return entry
+
+
+def _pick_base_view(model_name, datasets, relationships, hint):
+ """Choose the topic's base view: an explicit hint if given, else the dataset
+ that is never a relationship `to` (the FK sink of a many-to-one star)."""
+ if hint is not None:
+ if hint not in datasets:
+ raise ConversionError(
+ f"Model '{model_name}': requested base view '{hint}' is not a dataset")
+ return hint
+ if len(datasets) == 1:
+ return next(iter(datasets))
+ if not relationships:
+ raise ConversionError(
+ f"Model '{model_name}': {len(datasets)} datasets but no relationships; "
+ f"name the topic's base view with --base-view.")
+ incoming = {name: 0 for name in datasets}
+ for rel in relationships:
+ incoming[rel["to"]] += 1
+ roots = [n for n in datasets if incoming[n] == 0]
+ if not roots:
+ raise ConversionError(
+ f"Model '{model_name}': every dataset is a relationship target (the "
+ f"graph has a cycle); name the topic's base view with --base-view.")
+ if len(roots) > 1:
+ raise ConversionError(
+ f"Model '{model_name}': multiple candidate base views {sorted(roots)}; "
+ f"name the topic's base view with --base-view.")
+ return roots[0]
+
+
+def _build_topic(model, base_vname, view_names, relationships):
+ """Generate the topic for a fresh export: base view + the nested join map of
+ every view reachable from it, plus the model's description/AI context."""
+ topic = {"base_view": base_vname}
+ if model.get("description"):
+ topic["description"] = model["description"]
+ instructions = instructions_of(model.get("ai_context"))
+ if instructions:
+ topic["ai_context"] = instructions
+
+ # BFS the (undirected) relationship graph out from the base view; each view
+ # joins once, on its first-discovered (shortest) path. Views left unreachable
+ # simply stay out of the topic -- they are still exported and joinable, and
+ # every relationship remains in relationships.yaml either way.
+ adj = {}
+ for rel in relationships:
+ a, b = view_names[rel["from"]], view_names[rel["to"]]
+ adj.setdefault(a, []).append(b)
+ adj.setdefault(b, []).append(a)
+ children = {base_vname: {}}
+ seen = {base_vname}
+ queue = [base_vname]
+ while queue:
+ cur = queue.pop(0)
+ for neighbor in adj.get(cur, []):
+ if neighbor not in seen:
+ seen.add(neighbor)
+ children[cur][neighbor] = {}
+ children[neighbor] = children[cur][neighbor]
+ queue.append(neighbor)
+ if children[base_vname]:
+ topic["joins"] = children[base_vname]
+ return topic
+
+
+def _convert_metric(metric, resolve_base, view_names, dims_by_view, dialect):
+ """Map one OSI metric to an Omni measure. Returns (view_name, measure_name,
+ measure_dict), or None when the metric has no usable expression."""
+ mname_raw = require_str(metric, "name", "metric")
+ scope = f"metric '{mname_raw}'"
+ stash = read_stash(metric)
+ # The import qualifies a colliding measure name as `<view>__<name>` and
+ # stashes the original; restore it, else sanitize the OSI name.
+ mname = stash.get("name") or sanitize_name(mname_raw, scope, set())
+
+ if "measure" in stash:
+ # A prior import stashed the original Omni measure (a filtered, exotic,
+ # or otherwise non-reconstructible one) -- restore it verbatim and
+ # re-inject the natively-mapped properties.
+ measure = dict(stash["measure"])
+ _apply_metric_metadata(metric, measure)
+ return stash.get("view") or resolve_base(), mname, measure
+
+ expr = pick_expression(metric.get("expression"), dialect)
+ if expr is None:
+ _warn(scope, "no usable ANSI_SQL (or preferred-dialect) expression; "
+ "dropping metric")
+ return None
+
+ sanitized_views = set(view_names.values())
+ referenced = {
+ m.group(1)
+ for m in _DOTTED_REF_RE.finditer(expr)
+ if m.group(1) in sanitized_views
+ }
+
+ measure = None
+ target = None
+ m = _AGG_CALL_RE.match(expr)
+ if m and _balanced(m.group(2)):
+ func, inner = m.group(1).upper(), m.group(2).strip()
+ distinct = re.match(r"(?i)^DISTINCT\s+(.+)$", inner, re.DOTALL)
+ if func == "COUNT" and distinct:
+ func, inner = "COUNT_DISTINCT", distinct.group(1).strip()
+ agg = "count_distinct" if func == "COUNT_DISTINCT" else _AGG_TO_OMNI[func]
+ if func == "COUNT" and inner == "*":
+ measure, target = {"aggregate_type": "count"}, None
+ else:
+ dotted = _DOTTED_REF_RE.fullmatch(inner)
+ if dotted and dotted.group(1) in sanitized_views:
+ # AGG(view.name): a structured measure on that view. `name` may be
+ # a modeled field (referenced as ${name}) or a raw column covered
+ # by a field; only an unmodeled name stays a raw column ref.
+ vname, ref = dotted.group(1), dotted.group(2)
+ dims = dims_by_view.get(vname) or {"cols": {}, "names": set()}
+ dim = dims["cols"].get(ref) or (ref if ref in dims["names"] else None)
+ measure = {"sql": "${" + dim + "}" if dim else ref,
+ "aggregate_type": agg}
+ target = vname
+ elif not referenced:
+ # AGG over the base view's own columns (bare or computed).
+ measure = {"sql": inner, "aggregate_type": agg}
+ target = None
+ elif len(referenced) == 1:
+ measure = {"sql": osi_expr_refs_to_omni(inner, sanitized_views),
+ "aggregate_type": agg}
+ target = next(iter(referenced))
+
+ if measure is None:
+ # Anything else (a ratio, a multi-view aggregate, window SQL) becomes a
+ # raw-SQL measure; `view.col` references switch to `${view.col}` form.
+ target = next(iter(referenced)) if len(referenced) == 1 else None
+ measure = {"sql": osi_expr_refs_to_omni(expr, sanitized_views)}
+ if len(referenced) > 1:
+ _warn(scope, f"expression spans views {sorted(referenced)}; emitted as a "
+ f"raw-SQL measure on the base view -- verify the join path")
+
+ # Stashed placement (from import) wins; else the derived view; else base.
+ target = stash.get("view") or target or resolve_base()
+ _apply_metric_metadata(metric, measure)
+ if foreign_vendor_extensions(metric):
+ _warn(scope, "foreign-vendor custom_extensions dropped")
+ return target, mname, measure
+
+
+def _apply_metric_metadata(metric, measure):
+ if metric.get("description"):
+ measure["description"] = metric["description"]
+ syns = synonyms_of(metric.get("ai_context"))
+ if syns:
+ measure["synonyms"] = syns
+ instructions = instructions_of(metric.get("ai_context"))
+ if instructions:
+ measure["ai_context"] = instructions
+
+
+def _balanced(s):
+ depth = 0
+ for ch in s:
+ if ch == "(":
+ depth += 1
+ elif ch == ")":
+ depth -= 1
+ if depth < 0:
+ return False
+ return depth == 0
+
+
+def _warn_dropped_model(model):
+ if foreign_vendor_extensions(model):
+ _warn("model", "foreign-vendor custom_extensions dropped")
+ ai = model.get("ai_context")
+ if isinstance(ai, dict):
+ if ai.get("synonyms"):
+ _warn("model", "model ai_context synonyms have no Omni home; dropped")
+ if ai.get("examples"):
+ _warn("model", "model ai_context examples have no Omni home (Omni "
+ "sample_queries need a full query definition); dropped")
diff --git a/converters/omni/tests/_roundtrip_helpers.py b/converters/omni/tests/_roundtrip_helpers.py
new file mode 100644
index 0000000..bf003af
--- /dev/null
+++ b/converters/omni/tests/_roundtrip_helpers.py
@@ -0,0 +1,361 @@
+"""Shared model builders and round-trip assertions for property-based tests.
+
+This module is deliberately free of any third-party test dependency (no
+hypothesis, no pytest) so the generation + assertion logic can run two ways:
+
+ - driven by Hypothesis strategies (see test_roundtrip_properties.py), and
+ - driven by a plain seeded `random.Random` (RandomRnd below), which is how the
+ logic is exercised in environments where hypothesis is not installed.
+
+Both drivers implement the small `Rnd` interface (chance/count/pick/text/colname);
+the builders below depend only on that interface, so the generated model space is
+identical regardless of driver.
+
+The builders intentionally generate within the *round-trippable subset* -- the
+shapes the converter reproduces exactly. Known normalizations are avoided by
+construction (documented inline), e.g.:
+ - Omni names are generated already valid (lowercase snake_case), so the
+ sanitizer never renames anything;
+ - OSI metric expressions reference *field names* (`view.field`), which survive
+ the `${field}` modeled-reference trip; a raw column that differs from its
+ field's name would come back as the field name;
+ - OSI relationship names use the canonical `<from>_to_<to>` form the importer
+ regenerates.
+Name fuzzing (collisions, reserved words) is left to the targeted unit tests,
+which assert the converter *rejects* or *warns on* those inputs.
+"""
+
+import random
+import string
+import warnings
+
+from osi_omni import convert_omni_to_osi, convert_osi_to_omni
+from osi_omni._common import OSI_VERSION, dump_yaml, load_yaml
+
+from _util import strip_normalized
+
+_AGGS = ["sum", "average", "min", "max", "median", "count_distinct"]
+_OSI_AGGS = ["SUM", "AVG", "MIN", "MAX", "MEDIAN"]
+
+
+# --- Rnd backend for offline (no hypothesis) runs --------------------------------
+
+class RandomRnd:
+ """The `Rnd` interface backed by a seeded `random.Random`."""
+
+ def __init__(self, seed):
+ self.r = random.Random(seed)
+
+ def chance(self, p=0.5):
+ return self.r.random() < p
+
+ def count(self, lo, hi):
+ return self.r.randint(lo, hi)
+
+ def pick(self, seq):
+ return self.r.choice(list(seq))
+
+ def text(self):
+ # Alphanumeric with optional interior spaces; no leading/trailing space
+ # and no YAML-special characters, so the value is preserved verbatim
+ # through a dump/load cycle.
+ alnum = string.ascii_letters + string.digits
+ n = self.r.randint(0, 10)
+ body = "".join(self.r.choice(alnum + " ") for _ in range(n))
+ return (self.r.choice(alnum) + body).strip() or "x"
+
+ def colname(self):
+ first = self.r.choice(string.ascii_lowercase)
+ rest = "".join(
+ self.r.choice(string.ascii_lowercase + string.digits + "_")
+ for _ in range(self.r.randint(0, 7))
+ )
+ return first + rest
+
+
+class _Names:
+ """Hands out globally-unique names with a given prefix."""
+
+ def __init__(self):
+ self._n = {}
+
+ def next(self, prefix):
+ i = self._n.get(prefix, 0)
+ self._n[prefix] = i + 1
+ return f"{prefix}{i}"
+
+
+# --- Omni model builder (for Omni -> OSI -> Omni) ---------------------------------
+
+def _maybe_meta(rnd, target):
+ if rnd.chance(0.4):
+ target["description"] = rnd.text()
+ if rnd.chance(0.3):
+ target["label"] = rnd.text()
+ if rnd.chance(0.3):
+ target["synonyms"] = [rnd.text() for _ in range(rnd.count(1, 3))]
+ if rnd.chance(0.2):
+ target["ai_context"] = rnd.text()
+
+
+def _build_dimensions(rnd, names):
+ dims = {}
+ for _ in range(rnd.count(1, 4)):
+ dname = names.next("d")
+ dim = {}
+ flavor = rnd.count(0, 3)
+ if flavor == 1:
+ dim["sql"] = rnd.colname() # bare raw column
+ elif flavor == 2:
+ dim["sql"] = f'"{rnd.colname().upper()}"' # quoted identifier
+ elif flavor == 3 and dims:
+ dim["sql"] = "${" + rnd.pick(list(dims)) + "} + 1" # field ref
+ _maybe_meta(rnd, dim)
+ if rnd.chance(0.25):
+ dim["format"] = rnd.pick(["usdcurrency_2", "number_0", "percent_1", "id"])
+ if rnd.chance(0.2):
+ dim["hidden"] = True
+ if rnd.chance(0.2):
+ dim["group_label"] = rnd.text()
+ if rnd.chance(0.2):
+ dim["timeframes"] = ["raw", "date", "month"]
+ dims[dname] = dim
+ if rnd.chance(0.6):
+ # primary_key on a dimension whose sql is absent or a bare column.
+ for dname, dim in dims.items():
+ sql = dim.get("sql", "")
+ if "$" not in sql and '"' not in sql:
+ dim["primary_key"] = True
+ break
+ return dims
+
+
+def _build_measures(rnd, names, dims, shared_measure_name):
+ measures = {}
+ for _ in range(rnd.count(0, 3)):
+ mname = names.next("m")
+ flavor = rnd.count(0, 4)
+ if flavor == 0:
+ m = {"aggregate_type": "count"}
+ elif flavor == 1 and dims:
+ m = {"sql": "${" + rnd.pick(list(dims)) + "}",
+ "aggregate_type": rnd.pick(_AGGS)}
+ elif flavor == 2:
+ m = {"sql": rnd.colname(), "aggregate_type": rnd.pick(_AGGS)}
+ elif flavor == 3:
+ m = {"sql": f"SUM({rnd.colname()}) / 100"} # raw-SQL measure
+ else:
+ m = {"sql": rnd.colname(), "aggregate_type": "percentile",
+ "percentile": rnd.pick([50, 75, 95])}
+ if rnd.chance(0.3):
+ m["description"] = rnd.text()
+ if rnd.chance(0.2):
+ m["synonyms"] = [rnd.text() for _ in range(rnd.count(1, 2))]
+ if rnd.chance(0.2):
+ m["format"] = "usdcurrency_0"
+ if rnd.chance(0.2) and m.get("aggregate_type") == "count":
+ m["filters"] = {rnd.colname(): {"is": rnd.text()}}
+ measures[mname] = m
+ if shared_measure_name and rnd.chance(0.5):
+ # The same measure name on several views exercises the name-qualification
+ # (`view__measure`) and stashed-name restore paths.
+ measures[shared_measure_name] = {"aggregate_type": "count"}
+ return measures
+
+
+def build_omni(rnd):
+ """Generate an Omni model ({filename: YAML str}) in the round-trippable subset."""
+ names = _Names()
+ n_views = rnd.count(1, 4)
+ view_names = [names.next("v") for _ in range(n_views)]
+
+ files = {}
+ dims_of = {}
+ for vname in view_names:
+ view = {"schema": rnd.colname()}
+ if rnd.chance(0.4):
+ view["catalog"] = rnd.colname()
+ if rnd.chance(0.4):
+ view["table_name"] = rnd.colname().upper()
+ if rnd.chance(0.3):
+ view["description"] = rnd.text()
+ if rnd.chance(0.25):
+ view["label"] = rnd.text() # stash-only view extra
+ if rnd.chance(0.2):
+ view["tags"] = [rnd.colname()] # stash-only view extra
+ dims = _build_dimensions(rnd, names)
+ view["dimensions"] = dims
+ dims_of[vname] = dims
+ measures = _build_measures(rnd, names, dims, "count")
+ if measures:
+ view["measures"] = measures
+ files[f"views/{vname}.view.yaml"] = dump_yaml(view)
+
+ rels = []
+ for vname in view_names[1:]:
+ n_cols = rnd.count(1, 2)
+ clauses = [
+ "${" + f"{view_names[0]}.{rnd.colname()}" + "} = ${" + f"{vname}.{rnd.colname()}" + "}"
+ for _ in range(n_cols)
+ ]
+ rel = {"join_from_view": view_names[0], "join_to_view": vname,
+ "on_sql": " AND ".join(clauses),
+ "relationship_type": rnd.pick(
+ ["many_to_one", "many_to_one", "one_to_many", "one_to_one"])}
+ if rnd.chance(0.3):
+ rel["join_type"] = rnd.pick(["inner", "full_outer"])
+ if rnd.chance(0.2):
+ rel["reversible"] = True
+ if rnd.chance(0.2):
+ rel["where_sql"] = "${" + f"{vname}.{rnd.colname()}" + "}"
+ rels.append(rel)
+ if rels:
+ files["relationships.yaml"] = dump_yaml(rels)
+
+ if rnd.chance(0.7):
+ topic = {"base_view": view_names[0]}
+ if rnd.chance(0.5):
+ topic["description"] = rnd.text()
+ if rnd.chance(0.4):
+ topic["ai_context"] = rnd.text()
+ if rnd.chance(0.4):
+ topic["label"] = rnd.text()
+ if len(view_names) > 1 and rnd.chance(0.6):
+ topic["joins"] = {v: {} for v in view_names[1:]}
+ if rnd.chance(0.3):
+ topic["default_filters"] = {
+ f"{view_names[0]}.{rnd.colname()}": {"is": rnd.text()}}
+ files[f"topics/{names.next('t')}.topic.yaml"] = dump_yaml(topic)
+
+ if rnd.chance(0.4):
+ files["model.yaml"] = dump_yaml({
+ "week_start_day": rnd.pick(["Sunday", "Monday"]),
+ "included_schemas": [rnd.colname()],
+ })
+ return files
+
+
+# --- OSI model builder (for OSI -> Omni -> OSI) -----------------------------------
+
+def _osi_field(rnd, names):
+ fname = names.next("f")
+ flavor = rnd.count(0, 2)
+ if flavor == 0:
+ expr = fname # column named like the field
+ elif flavor == 1:
+ expr = rnd.colname() # renamed bare column
+ else:
+ expr = f"{rnd.colname()} || {rnd.colname()}" # computed expression
+ field = {"name": fname,
+ "expression": {"dialects": [{"dialect": "ANSI_SQL",
+ "expression": expr}]}}
+ if rnd.chance(0.4):
+ field["description"] = rnd.text()
+ if rnd.chance(0.3):
+ field["label"] = rnd.text()
+ ai = {}
+ if rnd.chance(0.3):
+ ai["synonyms"] = [rnd.text() for _ in range(rnd.count(1, 2))]
+ if rnd.chance(0.2):
+ ai["instructions"] = rnd.text()
+ if ai:
+ field["ai_context"] = ai
+ if rnd.chance(0.2):
+ field["dimension"] = {"is_time": True}
+ return field, expr
+
+
+def build_osi(rnd):
+ """Generate an OSI model dict in the round-trippable subset."""
+ names = _Names()
+ fact = names.next("fact")
+ dim_names = [names.next("dim") for _ in range(rnd.count(0, 3))]
+
+ datasets = []
+ fields_of = {}
+ for ds_name in [fact] + dim_names:
+ ds = {"name": ds_name,
+ "source": f"{rnd.colname()}.{rnd.colname()}.{rnd.colname()}"}
+ fields, simple_fields = [], []
+ for _ in range(rnd.count(1, 4)):
+ field, expr = _osi_field(rnd, names)
+ fields.append(field)
+ if expr == field["name"]:
+ simple_fields.append(field["name"])
+ if rnd.chance(0.3):
+ ds["description"] = rnd.text()
+ ds["fields"] = fields
+ # A primary key over a field whose column matches its name survives the
+ # dimension round-trip byte-for-byte.
+ if simple_fields and rnd.chance(0.5):
+ ds["primary_key"] = [simple_fields[0]]
+ datasets.append(ds)
+ fields_of[ds_name] = [f["name"] for f in fields]
+
+ relationships = []
+ for dname in dim_names:
+ n = rnd.count(1, 2)
+ relationships.append({
+ "name": f"{fact}_to_{dname}",
+ "from": fact, "to": dname,
+ "from_columns": [f"fk{i}_{rnd.colname()}" for i in range(n)],
+ "to_columns": [f"pk{i}_{rnd.colname()}" for i in range(n)],
+ })
+
+ metrics = []
+ for _ in range(rnd.count(0, 3)):
+ mname = names.next("metric")
+ flavor = rnd.count(0, 2)
+ target = rnd.pick([fact] + dim_names)
+ if flavor == 0:
+ expr = "COUNT(*)"
+ elif flavor == 1 and fields_of[target]:
+ expr = f"{rnd.pick(_OSI_AGGS)}({target}.{rnd.pick(fields_of[target])})"
+ else:
+ expr = f"SUM({fact}.{fields_of[fact][0]}) / COUNT(*)"
+ metric = {"name": mname,
+ "expression": {"dialects": [{"dialect": "ANSI_SQL",
+ "expression": expr}]}}
+ if rnd.chance(0.4):
+ metric["description"] = rnd.text()
+ if rnd.chance(0.3):
+ metric["ai_context"] = {"synonyms": [rnd.text()]}
+ metrics.append(metric)
+
+ model = {"name": names.next("model")}
+ if rnd.chance(0.4):
+ model["description"] = rnd.text()
+ if rnd.chance(0.3):
+ model["ai_context"] = {"instructions": rnd.text()}
+ model["datasets"] = datasets
+ if relationships:
+ model["relationships"] = relationships
+ if metrics:
+ model["metrics"] = metrics
+ return {"version": OSI_VERSION, "semantic_model": [model]}
+
+
+# --- Round-trip assertions -------------------------------------------------------
+
+def _quiet(fn, *args, **kwargs):
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ return fn(*args, **kwargs)
+
+
+def assert_omni_roundtrip(files):
+ """An Omni model survives Omni -> OSI -> Omni with every file identical
+ (structurally -- YAML key order and formatting aside)."""
+ osi = _quiet(convert_omni_to_osi, files)
+ files2 = _quiet(convert_osi_to_omni, osi)
+ parsed1 = {name: load_yaml(text, name) for name, text in files.items()}
+ parsed2 = {name: load_yaml(text, name) for name, text in files2.items()}
+ assert parsed2 == parsed1
+
+
+def assert_osi_roundtrip(osi):
+ """An OSI model dict survives OSI -> Omni -> OSI up to the documented
+ normalizations (see _util.strip_normalized)."""
+ files = _quiet(convert_osi_to_omni, dump_yaml(osi))
+ osi2 = load_yaml(_quiet(convert_omni_to_osi, files))
+ assert strip_normalized(osi2) == strip_normalized(osi)
diff --git a/converters/omni/tests/_util.py b/converters/omni/tests/_util.py
new file mode 100644
index 0000000..f0e2812
--- /dev/null
+++ b/converters/omni/tests/_util.py
@@ -0,0 +1,116 @@
+"""Shared test helpers: fixture loading and structural normalization."""
+
+import copy
+import json
+import pathlib
+
+from osi_omni._common import load_yaml # src is on sys.path via conftest.py
+
+FIXTURES = pathlib.Path(__file__).resolve().parent / "fixtures"
+REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]
+
+
+def load_fixture(name):
+ with open(FIXTURES / name) as fh:
+ return fh.read()
+
+
+def load_fixture_dir(name):
+ """Read a fixture Omni model directory as {relative posix path: text}."""
+ root = FIXTURES / name
+ files = {}
+ for path in sorted(root.rglob("*")):
+ if path.is_file():
+ files[path.relative_to(root).as_posix()] = path.read_text()
+ return files
+
+
+def parse(yaml_str):
+ return load_yaml(yaml_str)
+
+
+def parse_files(files):
+ """Parse every YAML file of an Omni model dict for structural comparison."""
+ return {name: load_yaml(text, name) for name, text in files.items()}
+
+
+def canon(obj):
+ """Deep-copy with every `custom_extensions[].data` JSON string parsed into a
+ dict, so comparisons are insensitive to JSON key order / whitespace."""
+ obj = copy.deepcopy(obj)
+
+ def walk(node):
+ if isinstance(node, dict):
+ for ext in node.get("custom_extensions") or []:
+ if isinstance(ext.get("data"), str):
+ ext["data"] = json.loads(ext["data"])
+ for v in node.values():
+ walk(v)
+ elif isinstance(node, list):
+ for v in node:
+ walk(v)
+
+ walk(obj)
+ return obj
+
+
+def strip_normalized(osi):
+ """Normalize away what the OSI -> Omni -> OSI trip changes by design, so a
+ round-trip comparison reflects the documented behavior:
+
+ - `custom_extensions` everywhere: the import adds OMNI stashes (topic set,
+ timeframe lists, ...) that a hand-authored source model does not carry.
+ - `unique_keys`: no Omni home (dropped with a warning on export).
+ - relationship `name`: regenerated as `<from>_to_<to>` on import.
+ - relationship `ai_context`: no Omni home (dropped with a warning).
+ - model `ai_context` synonyms/examples: no Omni home; only the
+ instructions text maps (onto the topic).
+ - `dimension: {is_time: false}`: equivalent to an absent `dimension`
+ (only `is_time: true` has an Omni form -- `timeframes`).
+ - a primary-key column no field covers materializes as a hidden
+ dimension on export, so it comes back as an extra (stash-only) field.
+ """
+ osi = copy.deepcopy(osi)
+ for model in osi.get("semantic_model", []):
+ model.pop("custom_extensions", None)
+ ai = model.get("ai_context")
+ if isinstance(ai, dict):
+ ai.pop("synonyms", None)
+ ai.pop("examples", None)
+ if not ai:
+ model.pop("ai_context")
+ for ds in model.get("datasets", []):
+ ds.pop("custom_extensions", None)
+ ds.pop("unique_keys", None)
+ ai = ds.get("ai_context")
+ if isinstance(ai, dict):
+ ai.pop("synonyms", None)
+ if not ai:
+ ds.pop("ai_context")
+ fields = ds.get("fields", []) or []
+ for field in fields:
+ field.pop("custom_extensions", None)
+ if field.get("dimension") == {"is_time": False}:
+ field.pop("dimension")
+ # Drop backfilled key fields: a bare-column field named after a
+ # primary_key column, carrying nothing but its expression.
+ pk = set(ds.get("primary_key") or [])
+ ds_fields = [
+ f for f in fields
+ if not (f["name"] in pk and set(f) <= {"name", "expression"})
+ ]
+ if ds_fields:
+ ds["fields"] = ds_fields
+ else:
+ ds.pop("fields", None)
+ for rel in model.get("relationships", []) or []:
+ rel["name"] = f"{rel['from']}_to_{rel['to']}"
+ rel.pop("ai_context", None)
+ rel.pop("custom_extensions", None)
+ for metric in model.get("metrics", []) or []:
+ metric.pop("custom_extensions", None)
+ # Metric order is not semantic; the import regroups metrics by the view
+ # their measure lives on.
+ if model.get("metrics"):
+ model["metrics"].sort(key=lambda m: m["name"])
+ return osi
diff --git a/converters/omni/tests/conftest.py b/converters/omni/tests/conftest.py
new file mode 100644
index 0000000..282f013
--- /dev/null
+++ b/converters/omni/tests/conftest.py
@@ -0,0 +1,6 @@
+import pathlib
+import sys
+
+# Make the converter modules in ../src importable from the tests.
+_SRC = pathlib.Path(__file__).resolve().parent.parent / "src"
+sys.path.insert(0, str(_SRC))
diff --git a/converters/omni/tests/fixtures/fixtureA_omni/relationships.yaml b/converters/omni/tests/fixtures/fixtureA_omni/relationships.yaml
new file mode 100644
index 0000000..04f7b21
--- /dev/null
+++ b/converters/omni/tests/fixtures/fixtureA_omni/relationships.yaml
@@ -0,0 +1,4 @@
+- join_from_view: orders
+ join_to_view: customer
+ on_sql: ${orders.o_custkey} = ${customer.c_custkey}
+ relationship_type: many_to_one
diff --git a/converters/omni/tests/fixtures/fixtureA_omni/topics/sales.topic.yaml b/converters/omni/tests/fixtures/fixtureA_omni/topics/sales.topic.yaml
new file mode 100644
index 0000000..6324461
--- /dev/null
+++ b/converters/omni/tests/fixtures/fixtureA_omni/topics/sales.topic.yaml
@@ -0,0 +1,5 @@
+base_view: orders
+description: Sales orders with customer attributes
+ai_context: Use this model for order analysis.
+joins:
+ customer: {}
diff --git a/converters/omni/tests/fixtures/fixtureA_omni/views/customer.view.yaml b/converters/omni/tests/fixtures/fixtureA_omni/views/customer.view.yaml
new file mode 100644
index 0000000..5706a4d
--- /dev/null
+++ b/converters/omni/tests/fixtures/fixtureA_omni/views/customer.view.yaml
@@ -0,0 +1,7 @@
+catalog: samples
+schema: tpch
+dimensions:
+ c_custkey:
+ primary_key: true
+ c_name:
+ description: Customer name
diff --git a/converters/omni/tests/fixtures/fixtureA_omni/views/orders.view.yaml b/converters/omni/tests/fixtures/fixtureA_omni/views/orders.view.yaml
new file mode 100644
index 0000000..f61582a
--- /dev/null
+++ b/converters/omni/tests/fixtures/fixtureA_omni/views/orders.view.yaml
@@ -0,0 +1,32 @@
+catalog: samples
+schema: tpch
+description: One row per order
+dimensions:
+ o_orderkey:
+ description: Order identifier
+ primary_key: true
+ o_custkey: {}
+ o_orderdate:
+ label: Order Date
+ synonyms:
+ - order date
+ - date
+ timeframes:
+ - raw
+ - date
+ - week
+ - month
+ - quarter
+ - year
+ o_totalprice: {}
+measures:
+ total_revenue:
+ sql: ${o_totalprice}
+ aggregate_type: sum
+ description: Total order revenue
+ synonyms:
+ - revenue
+ - sales
+ order_count:
+ aggregate_type: count
+ description: Number of orders
diff --git a/converters/omni/tests/fixtures/fixtureA_osi.yaml b/converters/omni/tests/fixtures/fixtureA_osi.yaml
new file mode 100644
index 0000000..b6e6863
--- /dev/null
+++ b/converters/omni/tests/fixtures/fixtureA_osi.yaml
@@ -0,0 +1,83 @@
+# yaml-language-server: $schema=../../../../core-spec/osi-schema.json
+#
+# Fixture A -- all-native round-trip (OSI -> Omni -> OSI).
+# Star schema; every construct maps to a native Omni slot. Documented
+# normalizations on the OSI -> Omni -> OSI trip: relationship names are
+# regenerated (`<from>_to_<to>`), and unique_keys that merely restate the
+# primary key are absorbed by it.
+
+version: "0.2.0.dev0"
+
+semantic_model:
+ - name: sales
+ description: Sales orders with customer attributes
+ ai_context:
+ instructions: "Use this model for order analysis."
+ datasets:
+ - name: orders # fact: no incoming relationship -> topic base_view
+ source: samples.tpch.orders
+ primary_key: [o_orderkey]
+ description: One row per order
+ fields:
+ - name: o_orderkey
+ expression:
+ dialects:
+ - dialect: ANSI_SQL
+ expression: o_orderkey
+ description: Order identifier
+ - name: o_custkey
+ expression:
+ dialects:
+ - dialect: ANSI_SQL
+ expression: o_custkey
+ - name: o_orderdate
+ expression:
+ dialects:
+ - dialect: ANSI_SQL
+ expression: o_orderdate
+ label: Order Date
+ dimension:
+ is_time: true
+ ai_context:
+ synonyms: [order date, date]
+ - name: o_totalprice
+ expression:
+ dialects:
+ - dialect: ANSI_SQL
+ expression: o_totalprice
+ - name: customer
+ source: samples.tpch.customer
+ primary_key: [c_custkey]
+ fields:
+ - name: c_custkey
+ expression:
+ dialects:
+ - dialect: ANSI_SQL
+ expression: c_custkey
+ - name: c_name
+ expression:
+ dialects:
+ - dialect: ANSI_SQL
+ expression: c_name
+ description: Customer name
+ relationships:
+ - name: orders_to_customer
+ from: orders
+ to: customer
+ from_columns: [o_custkey]
+ to_columns: [c_custkey]
+ metrics:
+ - name: total_revenue
+ expression:
+ dialects:
+ - dialect: ANSI_SQL
+ expression: SUM(orders.o_totalprice)
+ description: Total order revenue
+ ai_context:
+ synonyms: [revenue, sales]
+ - name: order_count
+ expression:
+ dialects:
+ - dialect: ANSI_SQL
+ expression: COUNT(*)
+ description: Number of orders
diff --git a/converters/omni/tests/fixtures/fixtureB_omni/model.yaml b/converters/omni/tests/fixtures/fixtureB_omni/model.yaml
new file mode 100644
index 0000000..d265896
--- /dev/null
+++ b/converters/omni/tests/fixtures/fixtureB_omni/model.yaml
@@ -0,0 +1,10 @@
+# Fixture B -- stash round-trip (Omni -> OSI -> Omni, lossless).
+# The model file has no OSI home and round-trips via the model-level stash.
+included_schemas:
+- ecomm
+week_start_day: Sunday
+access_grants:
+ pii_access:
+ user_attribute: role
+ allowed_values:
+ - admin
diff --git a/converters/omni/tests/fixtures/fixtureB_omni/relationships.yaml b/converters/omni/tests/fixtures/fixtureB_omni/relationships.yaml
new file mode 100644
index 0000000..cd91d26
--- /dev/null
+++ b/converters/omni/tests/fixtures/fixtureB_omni/relationships.yaml
@@ -0,0 +1,12 @@
+# Exercises: non-default join_type, reversible, a composite-key one_to_many
+# join (flipped to OSI's many-side-first orientation and flipped back on export).
+- join_from_view: order_items
+ join_to_view: users
+ on_sql: ${order_items.user_id} = ${users.id}
+ relationship_type: many_to_one
+ join_type: inner
+ reversible: true
+- join_from_view: users
+ join_to_view: orders
+ on_sql: ${users.id} = ${orders.user_id} AND ${users.region} = ${orders.region}
+ relationship_type: one_to_many
diff --git a/converters/omni/tests/fixtures/fixtureB_omni/topics/order_analysis.topic.yaml b/converters/omni/tests/fixtures/fixtureB_omni/topics/order_analysis.topic.yaml
new file mode 100644
index 0000000..57d0eae
--- /dev/null
+++ b/converters/omni/tests/fixtures/fixtureB_omni/topics/order_analysis.topic.yaml
@@ -0,0 +1,15 @@
+# Topic curation (label, joins, default_filters, fields) has no OSI home and
+# round-trips via the model-level stash; description/ai_context map natively.
+base_view: order_items
+label: Order Analysis
+description: Line-item order analysis
+ai_context: You are an ecommerce analyst.
+joins:
+ users:
+ orders: {}
+default_filters:
+ users.state:
+ is: California
+fields:
+- all_views.*
+- -users.state
diff --git a/converters/omni/tests/fixtures/fixtureB_omni/views/order_items.view.yaml b/converters/omni/tests/fixtures/fixtureB_omni/views/order_items.view.yaml
new file mode 100644
index 0000000..ca84597
--- /dev/null
+++ b/converters/omni/tests/fixtures/fixtureB_omni/views/order_items.view.yaml
@@ -0,0 +1,46 @@
+# Exercises the dimension/measure stash: format, hidden, group_label,
+# timeframes, convert_tz, a quoted-identifier sql, a ${field} reference, a
+# filtered measure, and a percentile measure.
+schema: ecomm
+table_name: ORDER_ITEMS
+label: Order Items
+description: One row per line item
+dimensions:
+ id:
+ primary_key: true
+ format: id
+ hidden: true
+ user_id: {}
+ sale_price:
+ sql: '"SALE_PRICE"'
+ format: usdcurrency_2
+ group_label: Money
+ created_at:
+ timeframes:
+ - raw
+ - date
+ - month
+ convert_tz: false
+ full_price:
+ sql: ${sale_price} * 1.1
+ label: Full Price
+measures:
+ count:
+ aggregate_type: count
+ total_revenue:
+ sql: ${sale_price}
+ aggregate_type: sum
+ format: usdcurrency_0
+ description: Total revenue
+ synonyms:
+ - sales
+ - revenue
+ completed:
+ aggregate_type: count
+ filters:
+ status:
+ is: complete
+ p95_price:
+ sql: ${sale_price}
+ aggregate_type: percentile
+ percentile: 95
diff --git a/converters/omni/tests/fixtures/fixtureB_omni/views/orders.view.yaml b/converters/omni/tests/fixtures/fixtureB_omni/views/orders.view.yaml
new file mode 100644
index 0000000..3264995
--- /dev/null
+++ b/converters/omni/tests/fixtures/fixtureB_omni/views/orders.view.yaml
@@ -0,0 +1,6 @@
+schema: ecomm
+dimensions:
+ order_id:
+ primary_key: true
+ user_id: {}
+ region: {}
diff --git a/converters/omni/tests/fixtures/fixtureB_omni/views/users.view.yaml b/converters/omni/tests/fixtures/fixtureB_omni/views/users.view.yaml
new file mode 100644
index 0000000..5b42d8b
--- /dev/null
+++ b/converters/omni/tests/fixtures/fixtureB_omni/views/users.view.yaml
@@ -0,0 +1,9 @@
+schema: ecomm
+dimensions:
+ id:
+ primary_key: true
+ region: {}
+ state: {}
+measures:
+ count:
+ aggregate_type: count
diff --git a/converters/omni/tests/fixtures/tpcds_omni/relationships.yaml b/converters/omni/tests/fixtures/tpcds_omni/relationships.yaml
new file mode 100644
index 0000000..4a70f2d
--- /dev/null
+++ b/converters/omni/tests/fixtures/tpcds_omni/relationships.yaml
@@ -0,0 +1,16 @@
+- join_from_view: store_sales
+ join_to_view: date_dim
+ on_sql: ${store_sales.ss_sold_date_sk} = ${date_dim.d_date_sk}
+ relationship_type: many_to_one
+- join_from_view: store_sales
+ join_to_view: customer
+ on_sql: ${store_sales.ss_customer_sk} = ${customer.c_customer_sk}
+ relationship_type: many_to_one
+- join_from_view: store_sales
+ join_to_view: item
+ on_sql: ${store_sales.ss_item_sk} = ${item.i_item_sk}
+ relationship_type: many_to_one
+- join_from_view: store_sales
+ join_to_view: store
+ on_sql: ${store_sales.ss_store_sk} = ${store.s_store_sk}
+ relationship_type: many_to_one
diff --git a/converters/omni/tests/fixtures/tpcds_omni/topics/tpcds_retail_model.topic.yaml b/converters/omni/tests/fixtures/tpcds_omni/topics/tpcds_retail_model.topic.yaml
new file mode 100644
index 0000000..51fcb06
--- /dev/null
+++ b/converters/omni/tests/fixtures/tpcds_omni/topics/tpcds_retail_model.topic.yaml
@@ -0,0 +1,11 @@
+base_view: store_sales
+description: TPC-DS retail semantic model for sales and customer analytics
+ai_context: Use this semantic model for retail analytics. It provides comprehensive
+ sales, customer, product, and store data from the TPC-DS benchmark. The model supports
+ time-based analysis, customer segmentation, product performance, and store operations
+ metrics.
+joins:
+ date_dim: {}
+ customer: {}
+ item: {}
+ store: {}
diff --git a/converters/omni/tests/fixtures/tpcds_omni/views/customer.view.yaml b/converters/omni/tests/fixtures/tpcds_omni/views/customer.view.yaml
new file mode 100644
index 0000000..c70c8c0
--- /dev/null
+++ b/converters/omni/tests/fixtures/tpcds_omni/views/customer.view.yaml
@@ -0,0 +1,27 @@
+catalog: tpcds
+schema: public
+description: Customer dimension with demographic information
+dimensions:
+ c_customer_sk:
+ description: Surrogate key for customer
+ primary_key: true
+ c_customer_id:
+ description: Business key for customer
+ synonyms:
+ - customer ID
+ - customer number
+ c_first_name:
+ description: Customer first name
+ c_last_name:
+ description: Customer last name
+ customer_full_name:
+ sql: c_first_name || ' ' || c_last_name
+ description: Customer full name (computed field)
+ synonyms:
+ - full name
+ - customer name
+ c_email_address:
+ description: Customer email address
+ synonyms:
+ - email
+ - contact
diff --git a/converters/omni/tests/fixtures/tpcds_omni/views/date_dim.view.yaml b/converters/omni/tests/fixtures/tpcds_omni/views/date_dim.view.yaml
new file mode 100644
index 0000000..dcc7160
--- /dev/null
+++ b/converters/omni/tests/fixtures/tpcds_omni/views/date_dim.view.yaml
@@ -0,0 +1,53 @@
+catalog: tpcds
+schema: public
+description: Date dimension with calendar attributes
+dimensions:
+ d_date_sk:
+ description: Surrogate key for date
+ primary_key: true
+ d_date:
+ description: Actual date value
+ synonyms:
+ - date
+ - calendar date
+ timeframes:
+ - raw
+ - date
+ - week
+ - month
+ - quarter
+ - year
+ d_year:
+ description: Year
+ synonyms:
+ - year
+ timeframes:
+ - raw
+ - date
+ - week
+ - month
+ - quarter
+ - year
+ d_quarter_name:
+ description: Quarter name (e.g., 2024Q1)
+ synonyms:
+ - quarter
+ - fiscal quarter
+ timeframes:
+ - raw
+ - date
+ - week
+ - month
+ - quarter
+ - year
+ d_month_name:
+ description: Month name
+ synonyms:
+ - month
+ timeframes:
+ - raw
+ - date
+ - week
+ - month
+ - quarter
+ - year
diff --git a/converters/omni/tests/fixtures/tpcds_omni/views/item.view.yaml b/converters/omni/tests/fixtures/tpcds_omni/views/item.view.yaml
new file mode 100644
index 0000000..c4fa8c2
--- /dev/null
+++ b/converters/omni/tests/fixtures/tpcds_omni/views/item.view.yaml
@@ -0,0 +1,33 @@
+catalog: tpcds
+schema: public
+description: Item/Product dimension with product attributes
+dimensions:
+ i_item_sk:
+ description: Surrogate key for item
+ primary_key: true
+ i_item_id:
+ description: Business key for item
+ synonyms:
+ - item ID
+ - product ID
+ - SKU
+ i_item_desc:
+ description: Item description
+ synonyms:
+ - product description
+ - item name
+ i_brand:
+ description: Brand name
+ synonyms:
+ - brand
+ - manufacturer
+ i_category:
+ description: Item category
+ synonyms:
+ - product category
+ - department
+ i_current_price:
+ description: Current price of the item
+ synonyms:
+ - price
+ - list price
diff --git a/converters/omni/tests/fixtures/tpcds_omni/views/store.view.yaml b/converters/omni/tests/fixtures/tpcds_omni/views/store.view.yaml
new file mode 100644
index 0000000..5a9520d
--- /dev/null
+++ b/converters/omni/tests/fixtures/tpcds_omni/views/store.view.yaml
@@ -0,0 +1,32 @@
+catalog: tpcds
+schema: public
+description: Store dimension with location and store attributes
+dimensions:
+ s_store_sk:
+ description: Surrogate key for store
+ primary_key: true
+ s_store_id:
+ description: Business key for store
+ synonyms:
+ - store ID
+ - store number
+ s_store_name:
+ description: Store name
+ synonyms:
+ - store name
+ - location name
+ s_city:
+ description: City where store is located
+ synonyms:
+ - city
+ - location
+ s_state:
+ description: State where store is located
+ synonyms:
+ - state
+ - region
+ s_number_employees:
+ description: Number of employees at the store
+ synonyms:
+ - employee count
+ - staff size
diff --git a/converters/omni/tests/fixtures/tpcds_omni/views/store_sales.view.yaml b/converters/omni/tests/fixtures/tpcds_omni/views/store_sales.view.yaml
new file mode 100644
index 0000000..0a21118
--- /dev/null
+++ b/converters/omni/tests/fixtures/tpcds_omni/views/store_sales.view.yaml
@@ -0,0 +1,90 @@
+catalog: tpcds
+schema: public
+description: Fact table containing all store sales transactions
+custom_compound_primary_key_sql:
+- ss_item_sk
+- ss_ticket_number
+dimensions:
+ ss_sold_date_sk:
+ description: Foreign key to date dimension
+ synonyms:
+ - sale date
+ - transaction date
+ ss_item_sk:
+ description: Foreign key to item dimension
+ synonyms:
+ - product
+ - item
+ ss_customer_sk:
+ description: Foreign key to customer dimension
+ synonyms:
+ - customer
+ - buyer
+ ss_store_sk:
+ description: Foreign key to store dimension
+ synonyms:
+ - store
+ - location
+ ss_quantity:
+ description: Quantity of items sold
+ synonyms:
+ - units sold
+ - quantity
+ ss_sales_price:
+ description: Sales price per unit
+ synonyms:
+ - unit price
+ - price
+ ss_ext_sales_price:
+ description: Extended sales price (quantity * price)
+ synonyms:
+ - total price
+ - line total
+ ss_net_profit:
+ description: Net profit from the sale
+ synonyms:
+ - profit
+ - margin
+ ss_ticket_number:
+ hidden: true
+measures:
+ total_sales:
+ sql: ${ss_ext_sales_price}
+ aggregate_type: sum
+ description: Total sales revenue across all transactions
+ synonyms:
+ - total revenue
+ - gross sales
+ - sales amount
+ total_profit:
+ sql: ${ss_net_profit}
+ aggregate_type: sum
+ description: Total net profit from store sales
+ synonyms:
+ - net profit
+ - total earnings
+ - profit
+ customer_lifetime_value:
+ sql: SUM(${store_sales.ss_ext_sales_price}) / COUNT(DISTINCT ${customer.c_customer_sk})
+ description: Average lifetime sales value per customer
+ synonyms:
+ - CLV
+ - LTV
+ - customer value
+ - lifetime revenue
+ sales_by_brand:
+ sql: ${ss_ext_sales_price}
+ aggregate_type: sum
+ description: Total sales by brand (requires grouping by item.i_brand)
+ synonyms:
+ - brand sales
+ - brand performance
+ - brand revenue
+ store_productivity:
+ sql: SUM(${store_sales.ss_ext_sales_price}) / NULLIF(SUM(${store.s_number_employees}),
+ 0)
+ description: Sales per employee across stores
+ synonyms:
+ - sales per employee
+ - employee productivity
+ - revenue per employee
diff --git a/converters/omni/tests/test_omni_to_osi.py b/converters/omni/tests/test_omni_to_osi.py
new file mode 100644
index 0000000..b124808
--- /dev/null
+++ b/converters/omni/tests/test_omni_to_osi.py
@@ -0,0 +1,376 @@
+"""Unit tests for the Omni -> OSI importer."""
+
+import json
+import warnings
+
+import pytest
+
+from osi_omni import ConversionError, convert_omni_to_osi
+from osi_omni._common import dump_yaml
+from _util import load_fixture_dir, parse
+
+
+def imp(files, **kwargs):
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ return parse(convert_omni_to_osi(files, **kwargs))["semantic_model"][0]
+
+
+def minimal_files(**view_overrides):
+ view = {"schema": "sch", "dimensions": {"id": {"primary_key": True},
+ "amount": {}}}
+ view.update(view_overrides)
+ return {"views/orders.view.yaml": dump_yaml(view)}
+
+
+def stash_of(obj):
+ for ext in obj.get("custom_extensions") or []:
+ if ext.get("vendor_name") == "OMNI":
+ return json.loads(ext["data"])
+ return {}
+
+
+def dataset(model, name):
+ return next(d for d in model["datasets"] if d["name"] == name)
+
+
+def field(ds, name):
+ return next(f for f in ds.get("fields", []) if f["name"] == name)
+
+
+def expr_of(obj):
+ return obj["expression"]["dialects"][0]["expression"]
+
+
+# --- structure --------------------------------------------------------------
+
+def test_view_becomes_dataset_with_source():
+ model = imp(minimal_files())
+ ds = dataset(model, "orders")
+ assert ds["source"] == "sch.orders" # table_name defaults to the view name
+ assert ds["primary_key"] == ["id"]
+ assert expr_of(field(ds, "amount")) == "amount"
+
+
+def test_catalog_and_table_name_join_into_source():
+ model = imp(minimal_files(catalog="db", table_name="RAW_ORDERS"))
+ assert dataset(model, "orders")["source"] == "db.sch.RAW_ORDERS"
+
+
+def test_sql_view_becomes_subquery_source():
+ files = {"views/v.view.yaml": dump_yaml({"sql": "SELECT 1 AS x",
+ "dimensions": {"x": {}}})}
+ assert dataset(imp(files), "v")["source"] == "SELECT 1 AS x"
+
+
+def test_view_without_schema_or_sql_is_stashed_not_converted():
+ # An extends-only view has no standalone dataset form; it is preserved in
+ # the model stash (and a model with nothing else convertible is an error).
+ files = dict(minimal_files())
+ files["views/v.view.yaml"] = dump_yaml({"extends": ["orders"],
+ "dimensions": {"x": {}}})
+ model = imp(files)
+ assert [d["name"] for d in model["datasets"]] == ["orders"]
+ assert "views/v.view.yaml" in stash_of(model)["extra_files"]
+
+ with pytest.raises(ConversionError, match="no convertible view files"):
+ imp({"views/v.view.yaml": dump_yaml({"dimensions": {"x": {}}})})
+
+
+def test_no_view_files_rejected():
+ with pytest.raises(ConversionError, match="no convertible view files"):
+ imp({"model.yaml": "{}\n"})
+
+
+def test_dimension_metadata_maps():
+ files = minimal_files(dimensions={
+ "amount": {"label": "Amount", "description": "The amount",
+ "synonyms": ["value"], "ai_context": "Prefer this."}})
+ f = field(dataset(imp(files), "orders"), "amount")
+ assert f["label"] == "Amount"
+ assert f["description"] == "The amount"
+ assert f["ai_context"] == {"instructions": "Prefer this.",
+ "synonyms": ["value"]}
+
+
+def test_timeframes_map_to_is_time_and_stash():
+ files = minimal_files(dimensions={"created_at": {"timeframes": ["raw", "date"]}})
+ f = field(dataset(imp(files), "orders"), "created_at")
+ assert f["dimension"] == {"is_time": True}
+ assert stash_of(f)["timeframes"] == ["raw", "date"]
+
+
+def test_field_reference_sql_translates_and_stashes_original():
+ files = minimal_files(dimensions={
+ "sale_price": {},
+ "full_price": {"sql": "${sale_price} * 1.1"}})
+ f = field(dataset(imp(files), "orders"), "full_price")
+ assert expr_of(f) == "sale_price * 1.1"
+ assert stash_of(f)["sql"] == "${sale_price} * 1.1"
+
+
+def test_table_ref_sql_translates():
+ files = minimal_files(dimensions={"x": {"sql": "${TABLE}.raw_x"}})
+ f = field(dataset(imp(files), "orders"), "x")
+ assert expr_of(f) == "raw_x"
+
+
+def test_omni_only_dimension_params_stash():
+ files = minimal_files(dimensions={
+ "amount": {"format": "usdcurrency_2", "hidden": True,
+ "group_label": "Money"}})
+ f = field(dataset(imp(files), "orders"), "amount")
+ assert stash_of(f) == {"_v": 1, "format": "usdcurrency_2", "hidden": True,
+ "group_label": "Money"}
+
+
+def test_compound_primary_key_resolves_to_columns():
+ files = minimal_files(
+ custom_compound_primary_key_sql=["id", "line"],
+ dimensions={"id": {}, "line": {"sql": "line_no"}})
+ assert dataset(imp(files), "orders")["primary_key"] == ["id", "line_no"]
+
+
+def test_view_extras_stash():
+ files = minimal_files(label="Orders", hidden=True, tags=["fact"])
+ ds = dataset(imp(files), "orders")
+ assert stash_of(ds)["view_extras"] == {"label": "Orders", "hidden": True,
+ "tags": ["fact"]}
+
+
+# --- relationships ----------------------------------------------------------
+
+def _two_view_files(rels):
+ return {
+ "views/orders.view.yaml": dump_yaml(
+ {"schema": "s", "dimensions": {"id": {"primary_key": True},
+ "user_id": {}}}),
+ "views/users.view.yaml": dump_yaml(
+ {"schema": "s", "dimensions": {"id": {"primary_key": True}}}),
+ "relationships.yaml": dump_yaml(rels),
+ }
+
+
+def test_many_to_one_join_decomposes():
+ model = imp(_two_view_files([
+ {"join_from_view": "orders", "join_to_view": "users",
+ "on_sql": "${orders.user_id} = ${users.id}",
+ "relationship_type": "many_to_one"}]))
+ rel = model["relationships"][0]
+ assert (rel["from"], rel["to"]) == ("orders", "users")
+ assert rel["from_columns"] == ["user_id"]
+ assert rel["to_columns"] == ["id"]
+ assert rel["name"] == "orders_to_users"
+
+
+def test_one_to_many_join_flips_to_many_side_first():
+ model = imp(_two_view_files([
+ {"join_from_view": "users", "join_to_view": "orders",
+ "on_sql": "${users.id} = ${orders.user_id}",
+ "relationship_type": "one_to_many"}]))
+ rel = model["relationships"][0]
+ assert (rel["from"], rel["to"]) == ("orders", "users")
+ assert rel["from_columns"] == ["user_id"]
+ assert rel["to_columns"] == ["id"]
+ assert stash_of(rel)["relationship_type"] == "one_to_many"
+
+
+def test_composite_on_sql_decomposes_in_order():
+ model = imp(_two_view_files([
+ {"join_from_view": "orders", "join_to_view": "users",
+ "on_sql": "${orders.a} = ${users.b} AND ${orders.c} = ${users.d}",
+ "relationship_type": "many_to_one"}]))
+ rel = model["relationships"][0]
+ assert rel["from_columns"] == ["a", "c"]
+ assert rel["to_columns"] == ["b", "d"]
+
+
+def test_field_reference_in_on_sql_resolves_to_column():
+ files = {
+ "views/orders.view.yaml": dump_yaml(
+ {"schema": "s",
+ "dimensions": {"user_key": {"sql": "user_id_raw"}}}),
+ "views/users.view.yaml": dump_yaml(
+ {"schema": "s", "dimensions": {"id": {}}}),
+ "relationships.yaml": dump_yaml([
+ {"join_from_view": "orders", "join_to_view": "users",
+ "on_sql": "${orders.user_key} = ${users.id}",
+ "relationship_type": "many_to_one"}]),
+ }
+ rel = imp(files)["relationships"][0]
+ assert rel["from_columns"] == ["user_id_raw"]
+
+
+def test_non_equi_join_is_stashed_not_converted():
+ # A range join is valid in Omni but has no OSI relationship form; it is
+ # preserved verbatim (with its position) rather than failing the import.
+ entry = {"join_from_view": "orders", "join_to_view": "users",
+ "on_sql": "${orders.total} >= ${users.threshold}",
+ "relationship_type": "many_to_one"}
+ model = imp(_two_view_files([entry]))
+ assert "relationships" not in model
+ assert stash_of(model)["extra_relationships"] == [
+ {"index": 0, "entry": entry}]
+
+
+def test_join_referencing_third_view_is_stashed_not_converted():
+ entry = {"join_from_view": "orders", "join_to_view": "users",
+ "on_sql": "${orders.x} = ${ghost.y}",
+ "relationship_type": "many_to_one"}
+ model = imp(_two_view_files([entry]))
+ assert "relationships" not in model
+ assert stash_of(model)["extra_relationships"] == [
+ {"index": 0, "entry": entry}]
+
+
+def test_join_to_missing_view_rejected():
+ with pytest.raises(ConversionError, match="has no view file"):
+ imp({"views/orders.view.yaml": dump_yaml({"schema": "s"}),
+ "relationships.yaml": dump_yaml([
+ {"join_from_view": "orders", "join_to_view": "ghost",
+ "on_sql": "${orders.x} = ${ghost.y}",
+ "relationship_type": "many_to_one"}])})
+
+
+def test_join_type_and_where_sql_stash():
+ model = imp(_two_view_files([
+ {"join_from_view": "orders", "join_to_view": "users",
+ "on_sql": "${orders.user_id} = ${users.id}",
+ "relationship_type": "many_to_one", "join_type": "inner",
+ "where_sql": "${users.active}", "reversible": True}]))
+ stash = stash_of(model["relationships"][0])
+ assert stash["join_type"] == "inner"
+ assert stash["where_sql"] == "${users.active}"
+ assert stash["reversible"] is True
+
+
+# --- measures ---------------------------------------------------------------
+
+def test_count_measure_becomes_count_star_metric():
+ files = minimal_files(measures={"count": {"aggregate_type": "count"}})
+ metric = imp(files)["metrics"][0]
+ assert expr_of(metric) == "COUNT(*)"
+ assert metric["name"] == "count"
+
+
+def test_sum_measure_qualifies_operand():
+ files = minimal_files(measures={
+ "total": {"sql": "${amount}", "aggregate_type": "sum",
+ "description": "Total", "synonyms": ["revenue"]}})
+ metric = imp(files)["metrics"][0]
+ assert expr_of(metric) == "SUM(orders.amount)"
+ assert metric["description"] == "Total"
+ assert metric["ai_context"] == {"synonyms": ["revenue"]}
+
+
+def test_count_distinct_measure():
+ files = minimal_files(measures={
+ "n_users": {"sql": "${amount}", "aggregate_type": "count_distinct"}})
+ assert expr_of(imp(files)["metrics"][0]) == "COUNT(DISTINCT orders.amount)"
+
+
+def test_raw_sql_measure_keeps_view_qualifiers():
+ files = minimal_files(measures={
+ "ratio": {"sql": "SUM(${orders.amount}) / COUNT(*)"}})
+ assert expr_of(imp(files)["metrics"][0]) == "SUM(orders.amount) / COUNT(*)"
+
+
+def test_filtered_measure_stashes_original():
+ files = minimal_files(measures={
+ "done": {"aggregate_type": "count",
+ "filters": {"status": {"is": "complete"}}}})
+ metric = imp(files)["metrics"][0]
+ assert expr_of(metric) == "COUNT(*)"
+ stash = stash_of(metric)
+ assert stash["measure"]["filters"] == {"status": {"is": "complete"}}
+ assert stash["view"] == "orders"
+
+
+def test_percentile_measure_best_effort_expression():
+ files = minimal_files(measures={
+ "p95": {"sql": "${amount}", "aggregate_type": "percentile",
+ "percentile": 95}})
+ metric = imp(files)["metrics"][0]
+ assert expr_of(metric) == (
+ "PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY orders.amount)")
+ assert stash_of(metric)["measure"]["aggregate_type"] == "percentile"
+
+
+def test_unknown_aggregate_type_rejected():
+ files = minimal_files(measures={"x": {"aggregate_type": "mystery"}})
+ with pytest.raises(ConversionError, match="unknown aggregate_type"):
+ imp(files)
+
+
+def test_colliding_measure_names_qualify_with_view():
+ files = {
+ "views/a.view.yaml": dump_yaml(
+ {"schema": "s", "measures": {"count": {"aggregate_type": "count"}}}),
+ "views/b.view.yaml": dump_yaml(
+ {"schema": "s", "measures": {"count": {"aggregate_type": "count"}}}),
+ }
+ names = sorted(m["name"] for m in imp(files)["metrics"])
+ assert names == ["a__count", "b__count"]
+
+
+# --- topics and the model file ----------------------------------------------
+
+def test_topic_maps_onto_model():
+ model = imp(load_fixture_dir("fixtureB_omni"))
+ assert model["name"] == "order_analysis"
+ assert model["description"] == "Line-item order analysis"
+ assert model["ai_context"] == {"instructions": "You are an ecommerce analyst."}
+ stash = stash_of(model)
+ assert stash["mapped_topic"] == "order_analysis"
+ assert stash["base_view"] == "order_items"
+ topic_stash = stash["topics"]["order_analysis"]
+ assert "description" not in topic_stash # natively mapped, not duplicated
+ assert topic_stash["label"] == "Order Analysis"
+ assert topic_stash["joins"] == {"users": {"orders": {}}}
+
+
+def test_model_file_stashes_verbatim():
+ model = imp(load_fixture_dir("fixtureB_omni"))
+ model_file = stash_of(model)["model_file"]
+ assert model_file["week_start_day"] == "Sunday"
+ assert model_file["included_schemas"] == ["ecomm"]
+
+
+def test_no_topics_stashes_empty_topic_set():
+ model = imp(minimal_files())
+ assert stash_of(model)["topics"] == {}
+
+
+def test_topic_flag_selects_mapped_topic():
+ files = dict(load_fixture_dir("fixtureB_omni"))
+ files["topics/second.topic.yaml"] = dump_yaml(
+ {"base_view": "users", "description": "Second"})
+ model = imp(files, topic="second")
+ assert model["description"] == "Second"
+
+
+def test_unknown_topic_flag_rejected():
+ with pytest.raises(ConversionError, match="not found"):
+ imp(load_fixture_dir("fixtureB_omni"), topic="ghost")
+
+
+def test_model_name_flag_overrides_topic_name():
+ model = imp(load_fixture_dir("fixtureB_omni"), model_name="my_model")
+ assert model["name"] == "my_model"
+
+
+def test_query_view_preserved_as_extra_file():
+ files = minimal_files()
+ files["views/facts.query.view.yaml"] = "schema: s\nsql: SELECT 1\n"
+ with warnings.catch_warnings(record=True) as ws:
+ warnings.simplefilter("always")
+ model = parse(convert_omni_to_osi(files))["semantic_model"][0]
+ assert any("query views" in str(w.message) for w in ws)
+ assert "views/facts.query.view.yaml" in stash_of(model)["extra_files"]
+
+
+def test_relationships_must_be_a_list():
+ files = minimal_files()
+ files["relationships.yaml"] = dump_yaml({"not": "a list"})
+ with pytest.raises(ConversionError, match="top-level YAML list"):
+ imp(files)
diff --git a/converters/omni/tests/test_osi_to_omni.py b/converters/omni/tests/test_osi_to_omni.py
new file mode 100644
index 0000000..ba0fb61
--- /dev/null
+++ b/converters/omni/tests/test_osi_to_omni.py
@@ -0,0 +1,334 @@
+"""Unit tests for the OSI -> Omni exporter."""
+
+import warnings
+
+import pytest
+
+from osi_omni import ConversionError, convert_osi_to_omni
+from osi_omni._common import dump_yaml
+from _util import REPO_ROOT, load_fixture, load_fixture_dir, parse, parse_files
+
+
+def export(osi_yaml, **kwargs):
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ return convert_osi_to_omni(osi_yaml, **kwargs)
+
+
+def minimal(**model_overrides):
+ model = {
+ "name": "m",
+ "datasets": [{"name": "orders", "source": "db.sch.orders",
+ "fields": [_field("amount")]}],
+ }
+ model.update(model_overrides)
+ return dump_yaml({"version": "0.2.0.dev0", "semantic_model": [model]})
+
+
+def _field(name, expr=None, dialect="ANSI_SQL", **extra):
+ f = {"name": name,
+ "expression": {"dialects": [{"dialect": dialect,
+ "expression": expr or name}]}}
+ f.update(extra)
+ return f
+
+
+# --- fixtures ---------------------------------------------------------------
+
+def test_fixtureA_export_matches_expected():
+ files = export(load_fixture("fixtureA_osi.yaml"))
+ assert parse_files(files) == parse_files(load_fixture_dir("fixtureA_omni"))
+
+
+def test_tpcds_export_matches_expected():
+ with open(REPO_ROOT / "examples" / "tpcds_semantic_model.yaml") as fh:
+ files = export(fh.read())
+ assert parse_files(files) == parse_files(load_fixture_dir("tpcds_omni"))
+
+
+# --- structure --------------------------------------------------------------
+
+def test_three_part_source_splits_into_catalog_schema_table():
+ files = export(minimal())
+ view = parse(files["views/orders.view.yaml"])
+ assert view["catalog"] == "db"
+ assert view["schema"] == "sch"
+ assert "table_name" not in view # table matches the view name
+
+
+def test_table_name_emitted_when_it_differs():
+ files = export(minimal(datasets=[{"name": "orders", "source": "sch.raw_orders"}]))
+ view = parse(files["views/orders.view.yaml"])
+ assert view == {"schema": "sch", "table_name": "raw_orders"}
+
+
+def test_subquery_source_becomes_sql_view():
+ files = export(minimal(datasets=[
+ {"name": "orders", "source": "SELECT * FROM t"}]))
+ assert parse(files["views/orders.view.yaml"])["sql"] == "SELECT * FROM t"
+
+
+def test_one_part_source_rejected():
+ with pytest.raises(ConversionError, match="no schema part"):
+ export(minimal(datasets=[{"name": "orders", "source": "orders"}]))
+
+
+def test_field_same_named_bare_column_gets_no_sql():
+ files = export(minimal())
+ assert parse(files["views/orders.view.yaml"])["dimensions"]["amount"] == {}
+
+
+def test_field_renamed_bare_column_gets_sql():
+ files = export(minimal(datasets=[
+ {"name": "orders", "source": "db.sch.orders",
+ "fields": [_field("total", "amount")]}]))
+ dims = parse(files["views/orders.view.yaml"])["dimensions"]
+ assert dims["total"] == {"sql": "amount"}
+
+
+def test_complex_expression_emitted_verbatim():
+ files = export(minimal(datasets=[
+ {"name": "orders", "source": "db.sch.orders",
+ "fields": [_field("full_name", "first || ' ' || last")]}]))
+ dims = parse(files["views/orders.view.yaml"])["dimensions"]
+ assert dims["full_name"]["sql"] == "first || ' ' || last"
+
+
+def test_is_time_maps_to_default_timeframes():
+ files = export(minimal(datasets=[
+ {"name": "orders", "source": "db.sch.orders",
+ "fields": [_field("created_at", dimension={"is_time": True})]}]))
+ dims = parse(files["views/orders.view.yaml"])["dimensions"]
+ assert dims["created_at"]["timeframes"] == [
+ "raw", "date", "week", "month", "quarter", "year"]
+
+
+def test_single_primary_key_marks_dimension():
+ files = export(minimal(datasets=[
+ {"name": "orders", "source": "db.sch.orders", "primary_key": ["id"],
+ "fields": [_field("id")]}]))
+ assert parse(files["views/orders.view.yaml"])["dimensions"]["id"][
+ "primary_key"] is True
+
+
+def test_composite_primary_key_uses_compound_key():
+ files = export(minimal(datasets=[
+ {"name": "orders", "source": "db.sch.orders",
+ "primary_key": ["id", "line"], "fields": [_field("id")]}]))
+ view = parse(files["views/orders.view.yaml"])
+ assert view["custom_compound_primary_key_sql"] == ["id", "line"]
+ # The uncovered key column materialized as a hidden dimension.
+ assert view["dimensions"]["line"] == {"hidden": True}
+
+
+def test_field_metadata_maps():
+ files = export(minimal(datasets=[
+ {"name": "orders", "source": "db.sch.orders",
+ "fields": [_field("amount", label="Amount", description="The amount",
+ ai_context={"synonyms": ["value"],
+ "instructions": "Prefer this."})]}]))
+ dim = parse(files["views/orders.view.yaml"])["dimensions"]["amount"]
+ assert dim["label"] == "Amount"
+ assert dim["description"] == "The amount"
+ assert dim["synonyms"] == ["value"]
+ assert dim["ai_context"] == "Prefer this."
+
+
+def test_relationship_becomes_join_entry():
+ files = export(minimal(datasets=[
+ {"name": "orders", "source": "db.sch.orders",
+ "fields": [_field("user_id")]},
+ {"name": "users", "source": "db.sch.users", "fields": [_field("id")]},
+ ], relationships=[
+ {"name": "r", "from": "orders", "to": "users",
+ "from_columns": ["user_id"], "to_columns": ["id"]}]))
+ rels = parse(files["relationships.yaml"])
+ assert rels == [{
+ "join_from_view": "orders", "join_to_view": "users",
+ "on_sql": "${orders.user_id} = ${users.id}",
+ "relationship_type": "many_to_one"}]
+
+
+def test_composite_relationship_joins_with_and():
+ files = export(minimal(datasets=[
+ {"name": "orders", "source": "db.sch.orders"},
+ {"name": "users", "source": "db.sch.users"},
+ ], relationships=[
+ {"name": "r", "from": "orders", "to": "users",
+ "from_columns": ["a", "b"], "to_columns": ["c", "d"]}]))
+ on_sql = parse(files["relationships.yaml"])[0]["on_sql"]
+ assert on_sql == "${orders.a} = ${users.c} AND ${orders.b} = ${users.d}"
+
+
+def test_topic_generated_with_fk_sink_base_and_join_tree():
+ files = export(load_fixture("fixtureA_osi.yaml"))
+ topic = parse(files["topics/sales.topic.yaml"])
+ assert topic["base_view"] == "orders"
+ assert topic["joins"] == {"customer": {}}
+ assert topic["description"] == "Sales orders with customer attributes"
+ assert topic["ai_context"] == "Use this model for order analysis."
+
+
+def test_explicit_base_view_overrides_heuristic():
+ files = export(load_fixture("fixtureA_osi.yaml"), base_view="customer")
+ topic = parse(files["topics/sales.topic.yaml"])
+ assert topic["base_view"] == "customer"
+ assert topic["joins"] == {"orders": {}}
+
+
+def test_unknown_base_view_rejected():
+ with pytest.raises(ConversionError, match="not a dataset"):
+ export(minimal(), base_view="nope")
+
+
+def test_multiple_roots_require_base_view():
+ osi = minimal(datasets=[{"name": "a", "source": "db.s.a"},
+ {"name": "b", "source": "db.s.b"}])
+ with pytest.raises(ConversionError, match="--base-view"):
+ export(osi)
+
+
+def test_metric_aggregate_becomes_structured_measure():
+ files = export(minimal(metrics=[
+ {"name": "total", "expression": {"dialects": [
+ {"dialect": "ANSI_SQL", "expression": "SUM(orders.amount)"}]}}]))
+ measures = parse(files["views/orders.view.yaml"])["measures"]
+ assert measures["total"] == {"sql": "${amount}", "aggregate_type": "sum"}
+
+
+def test_count_star_becomes_count_measure_on_base_view():
+ files = export(minimal(metrics=[
+ {"name": "n", "expression": {"dialects": [
+ {"dialect": "ANSI_SQL", "expression": "COUNT(*)"}]}}]))
+ assert parse(files["views/orders.view.yaml"])["measures"]["n"] == {
+ "aggregate_type": "count"}
+
+
+def test_count_distinct_metric():
+ files = export(minimal(metrics=[
+ {"name": "n", "expression": {"dialects": [
+ {"dialect": "ANSI_SQL",
+ "expression": "COUNT(DISTINCT orders.amount)"}]}}]))
+ assert parse(files["views/orders.view.yaml"])["measures"]["n"] == {
+ "sql": "${amount}", "aggregate_type": "count_distinct"}
+
+
+def test_ratio_metric_becomes_raw_sql_measure():
+ files = export(minimal(metrics=[
+ {"name": "avg_x", "expression": {"dialects": [
+ {"dialect": "ANSI_SQL",
+ "expression": "SUM(orders.amount) / COUNT(*)"}]}}]))
+ measure = parse(files["views/orders.view.yaml"])["measures"]["avg_x"]
+ assert measure == {"sql": "SUM(${orders.amount}) / COUNT(*)"}
+
+
+def test_dialect_preference():
+ field = {"name": "x", "expression": {"dialects": [
+ {"dialect": "ANSI_SQL", "expression": "ansi_col"},
+ {"dialect": "SNOWFLAKE", "expression": "snow_col"}]}}
+ files = export(minimal(datasets=[
+ {"name": "orders", "source": "db.sch.orders", "fields": [field]}]),
+ dialect="SNOWFLAKE")
+ assert parse(files["views/orders.view.yaml"])["dimensions"]["x"][
+ "sql"] == "snow_col"
+
+
+def test_names_are_sanitized():
+ files = export(dump_yaml({"version": "0.2.0.dev0", "semantic_model": [{
+ "name": "My Model",
+ "datasets": [{"name": "Order Items", "source": "db.sch.t",
+ "fields": [_field("Total Price", "p")]}]}]}))
+ assert "views/order_items.view.yaml" in files
+ assert "topics/my_model.topic.yaml" in files
+ dims = parse(files["views/order_items.view.yaml"])["dimensions"]
+ assert dims == {"total_price": {"sql": "p"}}
+
+
+def test_sanitization_collision_rejected():
+ osi = minimal(datasets=[{"name": "Orders", "source": "db.s.a"},
+ {"name": "orders", "source": "db.s.b"}])
+ with pytest.raises(ConversionError, match="collides"):
+ export(osi)
+
+
+def test_duplicate_metric_names_rejected():
+ metric = {"name": "total", "expression": {"dialects": [
+ {"dialect": "ANSI_SQL", "expression": "SUM(orders.amount)"}]}}
+ with pytest.raises(ConversionError, match="two metrics"):
+ export(minimal(metrics=[metric, dict(metric, name="Total")]))
+
+
+def test_relationship_column_count_mismatch_rejected():
+ osi = minimal(datasets=[{"name": "a", "source": "db.s.a"},
+ {"name": "b", "source": "db.s.b"}],
+ relationships=[{"name": "r", "from": "a", "to": "b",
+ "from_columns": ["x"], "to_columns": ["y", "z"]}])
+ with pytest.raises(ConversionError, match="same length"):
+ export(osi)
+
+
+def test_unknown_relationship_dataset_rejected():
+ osi = minimal(relationships=[{"name": "r", "from": "orders", "to": "ghost",
+ "from_columns": ["x"], "to_columns": ["y"]}])
+ with pytest.raises(ConversionError, match="unknown dataset"):
+ export(osi)
+
+
+def test_unsupported_version_rejected():
+ with pytest.raises(ConversionError, match="Unsupported OSI version"):
+ export(dump_yaml({"version": "9.9.9", "semantic_model": [{"name": "m"}]}))
+
+
+# --- warnings ---------------------------------------------------------------
+
+def _warnings_of(osi_yaml, **kwargs):
+ with warnings.catch_warnings(record=True) as ws:
+ warnings.simplefilter("always")
+ convert_osi_to_omni(osi_yaml, **kwargs)
+ return [str(w.message) for w in ws]
+
+
+def test_unique_keys_warn():
+ msgs = _warnings_of(minimal(datasets=[
+ {"name": "orders", "source": "db.s.t", "primary_key": ["id"],
+ "unique_keys": [["email"]], "fields": [_field("id")]}]))
+ assert any("unique_keys" in m for m in msgs)
+
+
+def test_unique_keys_restating_primary_key_do_not_warn():
+ msgs = _warnings_of(minimal(datasets=[
+ {"name": "orders", "source": "db.s.t", "primary_key": ["id"],
+ "unique_keys": [["id"]], "fields": [_field("id")]}]))
+ assert not any("unique_keys" in m for m in msgs)
+
+
+def test_field_without_usable_dialect_warns_and_drops():
+ osi = minimal(datasets=[
+ {"name": "orders", "source": "db.s.t",
+ "fields": [_field("x", dialect="MDX")]}])
+ msgs = _warnings_of(osi)
+ assert any("no usable" in m for m in msgs)
+ assert "dimensions" not in parse(export(osi)["views/orders.view.yaml"])
+
+
+def test_relationship_ai_context_warns():
+ msgs = _warnings_of(minimal(datasets=[
+ {"name": "a", "source": "db.s.a"}, {"name": "b", "source": "db.s.b"}],
+ relationships=[{"name": "r", "from": "a", "to": "b",
+ "from_columns": ["x"], "to_columns": ["y"],
+ "ai_context": {"synonyms": ["join"]}}]))
+ assert any("relationship ai_context" in m for m in msgs)
+
+
+def test_foreign_vendor_extensions_warn():
+ msgs = _warnings_of(minimal(custom_extensions=[
+ {"vendor_name": "DBT", "data": "{}"}]))
+ assert any("foreign-vendor" in m for m in msgs)
+
+
+def test_multiple_models_warn_and_first_converts():
+ osi = parse(minimal())
+ osi["semantic_model"].append({"name": "second", "datasets": [
+ {"name": "x", "source": "db.s.x"}]})
+ msgs = _warnings_of(dump_yaml(osi))
+ assert any("multiple semantic models" in m for m in msgs)
diff --git a/converters/omni/tests/test_real_world_layout.py b/converters/omni/tests/test_real_world_layout.py
new file mode 100644
index 0000000..b58a159
--- /dev/null
+++ b/converters/omni/tests/test_real_world_layout.py
@@ -0,0 +1,170 @@
+"""Round-trip tests for the model shape Omni's API/IDE actually emits.
+
+A model pulled from a real instance (`omni models yaml-get`) differs from the
+canonical `views/<name>.view.yaml` layout these converters write:
+
+ - view files live in per-schema folders (`DELIGHTED/response.view`) and the
+ model refers to them by a schema-qualified name (`delighted__response`)
+ recorded in a `# Reference this view as ...` header comment;
+ - joins restate Omni defaults (`join_type: always_left`,
+ `relationship_type: many_to_one`) and write compound keys / on_sql with
+ `${view.field}` references, mixed-case AND, and alias references;
+ - extends-only views, non-equi joins, template-syntax sql, empty-string
+ metadata, `timeframes: []`, camelCase/underscore-edged identifiers, and
+ schema names with spaces all occur.
+
+Every one of these was found in production models; the round trip through OSI
+must reproduce the original files exactly (same paths, same parsed content).
+"""
+
+import warnings
+
+import pytest
+
+from osi_omni import ConversionError, convert_omni_to_osi, convert_osi_to_omni
+from osi_omni._common import dump_yaml, load_yaml
+
+
+REAL_FILES = {
+ "model": dump_yaml({"included_schemas": ["DELIGHTED", "GITHUB"]}),
+ "relationships": dump_yaml([
+ # Omni defaults restated explicitly; ${view.field} refs; declared type.
+ {"join_from_view": "delighted__response",
+ "join_to_view": "delighted__person",
+ "join_type": "always_left",
+ "on_sql": "${delighted__response.person_id} = ${delighted__person.id}",
+ "relationship_type": "assumed_many_to_one"},
+ # Non-equi (range) join: valid Omni, no OSI form -- stashed.
+ {"join_from_view": "delighted__response",
+ "join_to_view": "github__team_membership",
+ "join_type": "always_left",
+ "on_sql": "${delighted__response.created_at} >= "
+ "${github__team_membership.valid_from}",
+ "relationship_type": "many_to_one"},
+ # Aliased join: on_sql references the alias, lowercase `and`.
+ {"join_from_view": "github__team_membership",
+ "join_to_view": "delighted__person",
+ "join_to_view_as": "membership_owner",
+ "on_sql": "${github__team_membership.user_id} = ${membership_owner.id} "
+ "and ${github__team_membership.org} = ${membership_owner.org}",
+ "relationship_type": "many_to_one"},
+ # Second join between the same pair (name dedup on the OSI side).
+ {"join_from_view": "github__team_membership",
+ "join_to_view": "delighted__person",
+ "on_sql": "${github__team_membership.approver_id} = "
+ "${delighted__person.id}",
+ "relationship_type": "many_to_one"},
+ ]),
+ "DELIGHTED/response.view": (
+ "# Reference this view as delighted__response\n" + dump_yaml({
+ "schema": "DELIGHTED", "table_name": "RESPONSE",
+ "description": "", # present-but-empty metadata survives
+ "custom_compound_primary_key_sql": [
+ "${delighted__response.id}", "${delighted__response.org}"],
+ "dimensions": {
+ "id": {"sql": '"ID"'},
+ "org": {"sql": '"ORG"'},
+ "person_id": {"sql": '"PERSON_ID"'},
+ "created_at": {"sql": '"CREATED_AT"', "timeframes": []},
+ "self_named": {"sql": "self_named"}, # explicit same-named col
+ "_fivetran_id": {"sql": '"_FIVETRAN_ID"'},
+ "payload_modelId": { # camelCase (JSON-flattened) name
+ "sql": "CAST(${delighted__response.payload}['modelId'] "
+ "AS VARCHAR)"},
+ "templated": {"sql": "CASE WHEN {{# delighted__response.f.filter }}"
+ " ${id} {{/ delighted__response.f.filter }} "
+ "THEN 1 ELSE 0 END"},
+ },
+ "measures": {"count": {"aggregate_type": "count"}},
+ })),
+ "DELIGHTED/person.view": (
+ "# Reference this view as delighted__person\n" + dump_yaml({
+ "schema": "DELIGHTED", # table_name implicit = file name
+ "dimensions": {"id": {"sql": '"ID"', "primary_key": True},
+ "org": {"sql": '"ORG"'}},
+ })),
+ "GITHUB/team_membership.view": (
+ "# Reference this view as github__team_membership\n" + dump_yaml({
+ "schema": "GITHUB", "table_name": "TEAM_MEMBERSHIP",
+ "dimensions": {"user_id": {}, "approver_id": {}, "org": {},
+ "valid_from": {}},
+ })),
+ # Extends-only view: no schema/sql of its own -- preserved, not converted.
+ "DELIGHTED/response_ext.view": (
+ "# Reference this view as delighted__response_ext\n" + dump_yaml({
+ "extends": ["delighted__response"],
+ "dimensions": {"extra": {"sql": "1"}},
+ })),
+ # Uploaded-CSV schema with a space in its name.
+ "Omni Views/upload.view": (
+ "# Reference this view as upload\n" + dump_yaml({
+ "schema": "Omni Views",
+ "uploaded_table_name": "upload.csv::abc",
+ "dimensions": {"user_id": {}},
+ })),
+ "Marketing/insights.topic": dump_yaml({
+ "base_view": "delighted__response",
+ "description": "Survey insights."}),
+}
+
+
+def _roundtrip(files):
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ osi_yaml = convert_omni_to_osi(files)
+ return osi_yaml, convert_osi_to_omni(osi_yaml)
+
+
+def test_api_layout_roundtrip_is_lossless():
+ _, exported = _roundtrip(REAL_FILES)
+ original = {k if "." in k.split("/")[-1] else k + ".yaml": v
+ for k, v in REAL_FILES.items()}
+ assert set(exported) == set(original)
+ for fname in original:
+ assert load_yaml(exported[fname]) == load_yaml(original[fname]), fname
+
+
+def test_qualified_view_names_come_from_reference_comment():
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ osi = load_yaml(convert_omni_to_osi(REAL_FILES))["semantic_model"][0]
+ names = {d["name"] for d in osi["datasets"]}
+ assert "delighted__response" in names
+ assert "upload" in names # comment name wins even with a folder
+ # The qualified name resolves relationship references and same-view
+ # ${view.field} compound-key entries.
+ rel = osi["relationships"][0]
+ assert rel["from"] == "delighted__response"
+ assert rel["to"] == "delighted__person"
+ response = next(d for d in osi["datasets"]
+ if d["name"] == "delighted__response")
+ assert response["primary_key"] == ["id", "org"]
+ assert response["source"] == "DELIGHTED.RESPONSE"
+ upload = next(d for d in osi["datasets"] if d["name"] == "upload")
+ assert upload["source"] == '"Omni Views".upload'
+
+
+def test_unmappable_omni_features_drop_out_of_osi_but_survive():
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ osi = load_yaml(convert_omni_to_osi(REAL_FILES))["semantic_model"][0]
+ # The extends view is not a dataset; the templated dimension not a field.
+ assert "delighted__response_ext" not in {d["name"] for d in osi["datasets"]}
+ response = next(d for d in osi["datasets"]
+ if d["name"] == "delighted__response")
+ assert "templated" not in {f["name"] for f in response["fields"]}
+ # The non-equi join is not an OSI relationship; the two same-pair joins get
+ # unique OSI names.
+ names = [r["name"] for r in osi["relationships"]]
+ assert len(names) == len(set(names)) == 3
+
+
+def test_duplicate_canonical_view_names_rejected():
+ files = {
+ "A/orders.view": "# Reference this view as orders\nschema: A\n",
+ "B/orders.view": "# Reference this view as orders\nschema: B\n",
+ }
+ with pytest.raises(ConversionError, match="two view files"):
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ convert_omni_to_osi(files)
diff --git a/converters/omni/tests/test_roundtrip.py b/converters/omni/tests/test_roundtrip.py
new file mode 100644
index 0000000..baccb52
--- /dev/null
+++ b/converters/omni/tests/test_roundtrip.py
@@ -0,0 +1,65 @@
+"""Fixture-based round-trip tests.
+
+- Omni -> OSI -> Omni must be lossless (the stash carries everything).
+- OSI -> Omni -> OSI must be identical up to the documented normalizations
+ (see _util.strip_normalized).
+- Every OSI document the importer emits must validate against the core-spec
+ JSON schema (skipped when jsonschema is not installed).
+"""
+
+import warnings
+
+import pytest
+
+from osi_omni import convert_omni_to_osi, convert_osi_to_omni
+from _util import (
+ REPO_ROOT,
+ load_fixture,
+ load_fixture_dir,
+ parse,
+ parse_files,
+ strip_normalized,
+)
+
+
+def _quiet(fn, *args, **kwargs):
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ return fn(*args, **kwargs)
+
+
+@pytest.mark.parametrize("fixture", ["fixtureA_omni", "fixtureB_omni", "tpcds_omni"])
+def test_omni_roundtrip_is_lossless(fixture):
+ files = load_fixture_dir(fixture)
+ osi = _quiet(convert_omni_to_osi, files)
+ files2 = _quiet(convert_osi_to_omni, osi)
+ assert parse_files(files2) == parse_files(files)
+
+
+@pytest.mark.parametrize("path", [
+ "fixtureA_osi.yaml",
+ str(REPO_ROOT / "examples" / "tpcds_semantic_model.yaml"),
+])
+def test_osi_roundtrip_up_to_documented_normalizations(path):
+ if "/" in path:
+ with open(path) as fh:
+ osi_yaml = fh.read()
+ else:
+ osi_yaml = load_fixture(path)
+ files = _quiet(convert_osi_to_omni, osi_yaml)
+ osi2 = _quiet(convert_omni_to_osi, files)
+ original = strip_normalized(parse(osi_yaml))
+ # Foreign-vendor extensions are dropped on export (documented); OSI-side
+ # normalization strips all custom_extensions from both sides.
+ assert strip_normalized(parse(osi2)) == original
+
+
+@pytest.mark.parametrize("fixture", ["fixtureA_omni", "fixtureB_omni", "tpcds_omni"])
+def test_imported_osi_validates_against_core_spec_schema(fixture):
+ jsonschema = pytest.importorskip("jsonschema")
+ import json
+
+ with open(REPO_ROOT / "core-spec" / "osi-schema.json") as fh:
+ schema = json.load(fh)
+ osi = _quiet(convert_omni_to_osi, load_fixture_dir(fixture))
+ jsonschema.validate(parse(osi), schema)
diff --git a/converters/omni/tests/test_roundtrip_properties.py b/converters/omni/tests/test_roundtrip_properties.py
new file mode 100644
index 0000000..445c1e2
--- /dev/null
+++ b/converters/omni/tests/test_roundtrip_properties.py
@@ -0,0 +1,83 @@
+"""Property-based round-trip tests.
+
+Two drivers over the same builders (see _roundtrip_helpers):
+
+ - Hypothesis, when installed: minimal failing examples and example databases.
+ - A plain seeded random.Random sweep otherwise, so the property logic still
+ runs in minimal environments.
+"""
+
+import pytest
+
+from _roundtrip_helpers import (
+ RandomRnd,
+ assert_omni_roundtrip,
+ assert_osi_roundtrip,
+ build_omni,
+ build_osi,
+)
+
+try:
+ from hypothesis import given, settings, strategies as st
+
+ HAVE_HYPOTHESIS = True
+except ImportError: # pragma: no cover
+ HAVE_HYPOTHESIS = False
+
+
+if HAVE_HYPOTHESIS:
+
+ class _HypRnd:
+ """The Rnd interface driven by hypothesis' `data` strategy."""
+
+ def __init__(self, data):
+ self.data = data
+
+ def chance(self, p=0.5):
+ return self.data.draw(st.floats(0, 1)) < p
+
+ def count(self, lo, hi):
+ return self.data.draw(st.integers(lo, hi))
+
+ def pick(self, seq):
+ return self.data.draw(st.sampled_from(list(seq)))
+
+ def text(self):
+ alnum = st.characters(whitelist_categories=("Lu", "Ll", "Nd"),
+ max_codepoint=0x7A)
+ head = self.data.draw(st.text(alphabet=alnum, min_size=1, max_size=1))
+ body = self.data.draw(st.text(
+ alphabet=st.sampled_from(
+ list("abcdefghijklmnopqrstuvwxyz"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 ")),
+ max_size=10))
+ return (head + body).strip()
+
+ def colname(self):
+ first = self.data.draw(st.sampled_from(
+ list("abcdefghijklmnopqrstuvwxyz")))
+ rest = self.data.draw(st.text(
+ alphabet=st.sampled_from(
+ list("abcdefghijklmnopqrstuvwxyz0123456789_")),
+ max_size=7))
+ return first + rest
+
+ @given(st.data())
+ @settings(max_examples=50, deadline=None)
+ def test_omni_roundtrip_property(data):
+ assert_omni_roundtrip(build_omni(_HypRnd(data)))
+
+ @given(st.data())
+ @settings(max_examples=50, deadline=None)
+ def test_osi_roundtrip_property(data):
+ assert_osi_roundtrip(build_osi(_HypRnd(data)))
+
+else: # pragma: no cover - exercised only without hypothesis
+
+ @pytest.mark.parametrize("seed", range(50))
+ def test_omni_roundtrip_seeded(seed):
+ assert_omni_roundtrip(build_omni(RandomRnd(seed)))
+
+ @pytest.mark.parametrize("seed", range(50))
+ def test_osi_roundtrip_seeded(seed):
+ assert_osi_roundtrip(build_osi(RandomRnd(seed)))
diff --git a/validation/validate.py b/validation/validate.py
index 5965115..258d34f 100644
--- a/validation/validate.py
+++ b/validation/validate.py
@@ -55,7 +55,7 @@ except ImportError:
try:
import sqlglot
- from sqlglot.errors import ParseError
+ from sqlglot.errors import ParseError, TokenError
SQLGLOT_AVAILABLE = True
except ImportError:
SQLGLOT_AVAILABLE = False
@@ -163,14 +163,14 @@ def validate_sql_expression(expr: str, dialect: str, context: str) -> str | None
# Try parsing as expression first (for field expressions like "column_name")
sqlglot.parse_one(expr, dialect=sqlglot_dialect)
return None
- except ParseError:
+ except (ParseError, TokenError):
pass
try:
# Try wrapping in SELECT for simple column references
sqlglot.parse_one(f"SELECT {expr}", dialect=sqlglot_dialect)
return None
- except ParseError as e:
+ except (ParseError, TokenError) as e:
return f"[SQL] {context}: {str(e).split(chr(10))[0]}"