You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sourcelume.apache.org by an...@apache.org on 2026/09/07 18:21:04 UTC
(sourcelume-spec) branch minimal-mimir created (now 8f04282)
This is an automated email from the ASF dual-hosted git repository.
ANierbeck pushed a change to branch minimal-mimir
in repository https://gitbox.apache.org/repos/asf/sourcelume-spec.git
at 8f04282 Minimal 0.0.1 record shape, validated against the DFM Mimir danish-pd corpus
This branch includes the following new commits:
new 8f04282 Minimal 0.0.1 record shape, validated against the DFM Mimir danish-pd corpus
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@sourcelume.apache.org
For additional commands, e-mail: commits-help@sourcelume.apache.org
(sourcelume-spec) 01/01: Minimal 0.0.1 record shape, validated against the DFM Mimir danish-pd corpus
Posted by an...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ANierbeck pushed a commit to branch minimal-mimir
in repository https://gitbox.apache.org/repos/asf/sourcelume-spec.git
commit 8f0428275368f7280b1ec692c6d2cc00d294f88b
Author: anierbeck <ac...@codecentric.de>
AuthorDate: Sun Sep 6 19:54:36 2026 +0200
Minimal 0.0.1 record shape, validated against the DFM Mimir danish-pd corpus
Fills in the 0.0.1 schema/context/SHACL/prose that arrived as empty
skeletons on origin/main, and proves the shape works by expressing one
real source dataset from DFM Mimir's training corpus as a
ProvenanceRecord that validates (jsonschema + pyshacl + format
assertions, not just JSON-parse).
Record shape (0.0.1):
- creator: array (min 1), each entry optionally role-tagged
(originator/curator/distributor) -- a single creator still validates.
- custodyChain: array (min 1), ordered, earliest first -- the
minimum that can represent a multi-hop chain at all.
- license: IRI, with the CC Public Domain Mark IRI documented in the
spec prose as the interim convention for public-domain-by-expiration
claims (SPDX has no direct term).
- no signature block (signing lives in Sourcelume Attest, per the
site's own pipeline description).
Example: examples/dfm-mimir-danish-pd-best-effort.jsonld -- danish-pd
(Danish monographs whose copyright expired), aggregated by LANGU:IA
(March 2024), repackaged by Danish Foundation Models.
Validator: tools/validate.py upgraded from skeleton (parse-only) to
real jsonschema 2020-12 + pyshacl + format assertions. pyproject.toml
needs jsonschema[format] so format: uri is actually enforced.
Deliberately out of scope for this change: crosswalks to Croissant/OTDI/
SPDX, the stress-test memo, a minimal-record example, CI wiring --
those belong in follow-ups once the record shape is agreed on the list.
Apache-ai: Yes
Generated-by: Claude Code (Sonnet 5)
Reviewed-by: anierbeck
---
context/0.0.1/sourcelume.jsonld | 41 ++++--
examples/dfm-mimir-danish-pd-best-effort.jsonld | 35 +++++
pyproject.toml | 4 +-
schema/0.0.1/sourcelume.schema.json | 94 ++++++++++--
schema/0.0.1/sourcelume.shacl.ttl | 69 +++++----
spec/0.0.1/index.md | 128 +++++++---------
tools/validate.py | 185 ++++++------------------
uv.lock | 139 +++++++++++++++++-
8 files changed, 419 insertions(+), 276 deletions(-)
diff --git a/context/0.0.1/sourcelume.jsonld b/context/0.0.1/sourcelume.jsonld
index 35abea4..ce58627 100644
--- a/context/0.0.1/sourcelume.jsonld
+++ b/context/0.0.1/sourcelume.jsonld
@@ -1,16 +1,39 @@
{
"@context": {
- "@version": 1.1,
- "sl": "https://sourcelume.apache.org/ns/0.0.1#",
+ "sourcelume": "https://sourcelume.apache.org/ns#",
+ "dct": "http://purl.org/dc/terms/",
+ "schema": "https://schema.org/",
+ "spdx": "http://spdx.org/rdf/terms#",
+ "xsd": "http://www.w3.org/2001/XMLSchema#",
+
"id": "@id",
"type": "@type",
- "name": "sl:name",
- "description": "sl:description",
- "version": "sl:version",
- "createdAt": {
- "@id": "sl:createdAt",
- "@type": "http://www.w3.org/2001/XMLSchema#dateTime"
+ "ProvenanceRecord": "sourcelume:ProvenanceRecord",
+
+ "identifier": "dct:identifier",
+ "name": "schema:name",
+ "version": "schema:version",
+ "license": {
+ "@id": "dct:license",
+ "@type": "@id"
+ },
+ "creator": "schema:creator",
+ "role": "sourcelume:role",
+ "created": {
+ "@id": "dct:created",
+ "@type": "xsd:dateTime"
+ },
+ "origin": "sourcelume:origin",
+ "custodyChain": "sourcelume:custodyChain",
+ "agent": {
+ "@id": "schema:agent",
+ "@type": "@id"
+ },
+ "action": "sourcelume:action",
+ "startTime": {
+ "@id": "schema:startTime",
+ "@type": "xsd:dateTime"
}
}
-}
\ No newline at end of file
+}
diff --git a/examples/dfm-mimir-danish-pd-best-effort.jsonld b/examples/dfm-mimir-danish-pd-best-effort.jsonld
new file mode 100644
index 0000000..a917d66
--- /dev/null
+++ b/examples/dfm-mimir-danish-pd-best-effort.jsonld
@@ -0,0 +1,35 @@
+{
+ "@context": "../context/0.0.1/sourcelume.jsonld",
+ "id": "https://sourcelume.apache.org/records/dfm-mimir-v1/danish-pd",
+ "type": "ProvenanceRecord",
+ "identifier": "https://huggingface.co/datasets/PleIAs/Danish-PD",
+ "name": "PleIAs - Danish Public Domain",
+ "version": "danish-pd@dfm-dynaword-v1",
+ "license": "https://creativecommons.org/publicdomain/mark/1.0/",
+ "creator": [
+ {
+ "type": "schema:Organization",
+ "name": "LANGU:IA",
+ "role": "curator"
+ },
+ {
+ "type": "schema:Organization",
+ "name": "PleIAs",
+ "role": "distributor"
+ }
+ ],
+ "created": "2026-09-05T00:00:00Z",
+ "origin": "3113 Danish monographs/periodicals whose copyright has expired, recovered from Internet Archive and various European national libraries and cultural heritage institutions. The original authors/rightsholders are not individually identifiable at this scale and are omitted from creator/custodyChain; see this field for that gap.",
+ "custodyChain": [
+ {
+ "agent": "https://huggingface.co/datasets/PleIAs",
+ "action": "aggregated",
+ "startTime": "2024-03-01T00:00:00Z"
+ },
+ {
+ "agent": "https://huggingface.co/danish-foundation-models",
+ "action": "repackaged-into-training-corpus",
+ "startTime": "2026-01-01T00:00:00Z"
+ }
+ ]
+}
diff --git a/pyproject.toml b/pyproject.toml
index 69f6a23..bf68668 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -31,7 +31,9 @@ dependencies = []
[project.optional-dependencies]
dev = [
- "jsonschema",
+ # [format] pulls in rfc3987, needed so jsonschema actually enforces
+ # "format": "uri" instead of silently treating it as a no-op annotation.
+ "jsonschema[format]",
"pyld",
"rdflib",
"pyshacl",
diff --git a/schema/0.0.1/sourcelume.schema.json b/schema/0.0.1/sourcelume.schema.json
index 5a682f7..f1542ca 100644
--- a/schema/0.0.1/sourcelume.schema.json
+++ b/schema/0.0.1/sourcelume.schema.json
@@ -1,36 +1,102 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://sourcelume.apache.org/schema/0.0.1/sourcelume.schema.json",
- "title": "Apache Sourcelume Record",
- "description": "Minimal skeleton schema for an Apache Sourcelume record. Fields and constraints will be filled in as the specification develops.",
+ "title": "Apache Sourcelume Provenance Record (0.0.1)",
+ "description": "Minimal viable structural schema for a Sourcelume ProvenanceRecord: dataset identity, one or more role-tagged creators, one license claim, and an ordered custody chain. No signature block yet.",
"type": "object",
+ "required": [
+ "id",
+ "type",
+ "identifier",
+ "name",
+ "license",
+ "creator",
+ "created",
+ "origin",
+ "custodyChain"
+ ],
"properties": {
+ "@context": {},
"id": {
"type": "string",
- "description": "Unique identifier (IRI) for this record."
+ "format": "uri"
},
"type": {
+ "const": "ProvenanceRecord"
+ },
+ "identifier": {
"type": "string",
- "description": "The record type."
+ "minLength": 1,
+ "description": "Identifier for the dataset being described (e.g. a DOI or persistent URL), not for this record."
},
"name": {
"type": "string",
- "description": "Human-readable name for the record."
+ "minLength": 1
+ },
+ "version": {
+ "type": "string"
},
- "description": {
+ "license": {
"type": "string",
- "description": "Free-text description of the record."
+ "format": "uri",
+ "description": "IRI of the claimed license (e.g. an SPDX license page URL)."
},
- "version": {
+ "creator": {
+ "type": "array",
+ "minItems": 1,
+ "items": {
+ "type": "object",
+ "required": ["type", "name"],
+ "properties": {
+ "type": {
+ "enum": ["schema:Organization", "schema:Person"]
+ },
+ "name": {
+ "type": "string",
+ "minLength": 1
+ },
+ "role": {
+ "type": "string",
+ "enum": ["originator", "curator", "distributor"],
+ "description": "Optional. Distinguishes who created the underlying data from who curated or redistributed it. Omit if the record has a single, undifferentiated creator."
+ }
+ },
+ "additionalProperties": true
+ }
+ },
+ "created": {
"type": "string",
- "description": "Version of the record content, e.g. following semver."
+ "format": "date-time"
},
- "createdAt": {
+ "origin": {
"type": "string",
- "format": "date-time",
- "description": "Creation timestamp, ISO 8601."
+ "minLength": 1,
+ "description": "Free-text description of where the dataset's underlying data came from."
+ },
+ "custodyChain": {
+ "type": "array",
+ "minItems": 1,
+ "description": "Ordered chain of custody events, earliest first. A single-hop dataset has exactly one entry.",
+ "items": {
+ "type": "object",
+ "required": ["agent", "action", "startTime"],
+ "properties": {
+ "agent": {
+ "type": "string",
+ "format": "uri"
+ },
+ "action": {
+ "type": "string",
+ "minLength": 1
+ },
+ "startTime": {
+ "type": "string",
+ "format": "date-time"
+ }
+ },
+ "additionalProperties": true
+ }
}
},
- "required": ["id", "type"],
"additionalProperties": true
-}
\ No newline at end of file
+}
diff --git a/schema/0.0.1/sourcelume.shacl.ttl b/schema/0.0.1/sourcelume.shacl.ttl
index ac577f0..fe0e4b0 100644
--- a/schema/0.0.1/sourcelume.shacl.ttl
+++ b/schema/0.0.1/sourcelume.shacl.ttl
@@ -1,52 +1,49 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
@prefix sh: <http://www.w3.org/ns/shacl#> .
-@prefix sl: <https://sourcelume.apache.org/ns/0.0.1#> .
+@prefix sourcelume: <https://sourcelume.apache.org/ns#> .
+@prefix dct: <http://purl.org/dc/terms/> .
+@prefix schema: <https://schema.org/> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
-# Minimal skeleton SHACL shape for a Sourcelume record.
-# Constraints will be expanded as the specification develops, and should
-# stay aligned with schema/0.0.1/sourcelume.schema.json.
-
-sl:RecordShape
+sourcelume:ProvenanceRecordShape
a sh:NodeShape ;
- sh:targetClass sl:Record ;
-
+ sh:targetClass sourcelume:ProvenanceRecord ;
sh:property [
- sh:path sl:name ;
- sh:datatype xsd:string ;
+ sh:path dct:identifier ;
+ sh:minCount 1 ;
sh:maxCount 1 ;
+ sh:datatype xsd:string ;
] ;
-
sh:property [
- sh:path sl:description ;
- sh:datatype xsd:string ;
+ sh:path schema:name ;
+ sh:minCount 1 ;
sh:maxCount 1 ;
+ sh:datatype xsd:string ;
] ;
-
sh:property [
- sh:path sl:version ;
- sh:datatype xsd:string ;
+ sh:path dct:license ;
+ sh:minCount 1 ;
sh:maxCount 1 ;
+ sh:nodeKind sh:IRI ;
+ ] ;
+ sh:property [
+ sh:path schema:creator ;
+ sh:minCount 1 ;
+ sh:nodeKind sh:BlankNodeOrIRI ;
] ;
-
sh:property [
- sh:path sl:createdAt ;
+ sh:path dct:created ;
+ sh:minCount 1 ;
+ sh:maxCount 1 ;
sh:datatype xsd:dateTime ;
+ ] ;
+ sh:property [
+ sh:path sourcelume:origin ;
+ sh:minCount 1 ;
sh:maxCount 1 ;
- ] .
\ No newline at end of file
+ sh:datatype xsd:string ;
+ ] ;
+ sh:property [
+ sh:path sourcelume:custodyChain ;
+ sh:minCount 1 ;
+ sh:nodeKind sh:BlankNodeOrIRI ;
+ ] .
diff --git a/spec/0.0.1/index.md b/spec/0.0.1/index.md
index a2dd094..5fd4782 100644
--- a/spec/0.0.1/index.md
+++ b/spec/0.0.1/index.md
@@ -1,74 +1,54 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-
-# Apache Sourcelume Specification — v0.0.1
-
-> **Status:** Draft / skeleton. This version is a starting point for
-> development and has not been released. Content will be filled in as the
-> specification develops. See `VERSIONING.md` for how this version number
-> relates to changes in `context/` and `schema/`.
-
-## 1. Introduction
-
-Sourcelume defines a vocabulary and data model for describing
-[fill in: what Sourcelume records represent, and the problem the spec
-solves].
-
-## 2. Terminology
-
-This section defines terms used throughout the specification.
-
-- **Record** — [definition]
-
-## 3. Data model
-
-This section describes the structure of a Sourcelume record.
-
-Normative field definitions live in the corresponding JSON Schema
-(`schema/0.0.1/sourcelume.schema.json`) and SHACL shapes
-(`schema/0.0.1/sourcelume.shacl.ttl`). This document should describe the
-*meaning* and intended usage of each field; the schema/shapes describe the
-*constraints*.
-
-| Field | Type | Required | Description |
-|---------------|----------|----------|---------------------------------------|
-| `id` | IRI | Yes | Unique identifier for the record. |
-| `type` | string | Yes | The record type. |
-| `name` | string | No | Human-readable name. |
-| `description` | string | No | Free-text description. |
-| `version` | string | No | Version of the record content. |
-| `createdAt` | datetime | No | Creation timestamp (ISO 8601). |
-
-## 4. JSON-LD context
-
-Sourcelume records are expressed as JSON-LD using the context published at
-`context/0.0.1/sourcelume.jsonld`.
-
-## 5. Examples
-
-See `examples/` for sample records conforming to this version of the
-specification.
-
-## 6. Conformance
-
-[fill in: what it means for a document/tool to conform to this version of
-the spec]
-
-## 7. Changes from previous versions
-
-This is the first version of the specification; there is no prior version
-to compare against.
\ No newline at end of file
+# Apache Sourcelume Provenance Record — 0.0.1
+
+## Status
+
+Draft. Describes the `ProvenanceRecord` type defined by schema/context version `0.0.1`. This is
+the minimal viable shape: enough to publish a checkable claim about one dataset's identity, one
+or more role-tagged creators, its license, its origin, and an ordered custody chain. It
+intentionally leaves out cryptographic attestation and downstream usage/audit metadata — those
+are planned for later, once Sourcelume Attest exists and once the dataset-vs-model-usage scope
+question is settled on the dev list (see `notes/stress-test-dfm-mimir.md`).
+
+## Purpose
+
+A `ProvenanceRecord` is a structured, machine-readable claim of the form: *this dataset, with
+this identity, was produced/curated/distributed by these parties, is licensed under these terms,
+came from this origin, and passed through this chain of custody.* Sourcelume does not verify the
+claim's accuracy — it gives the claim a consistent shape so it *can* be verified by someone else.
+
+## Fields
+
+| Field | Required | Type | Description |
+|---|---|---|---|
+| `id` | yes | IRI | Identifier for this provenance record itself (not necessarily the dataset). |
+| `type` | yes | `"ProvenanceRecord"` | Fixed type discriminator. |
+| `identifier` | yes | string (IRI or DOI) | Identifier for the dataset being described. |
+| `name` | yes | string | Human-readable dataset name. |
+| `version` | no | string | Dataset version, if versioned. |
+| `license` | yes | IRI | License under which the dataset is claimed to be distributed (e.g. an SPDX license URL). For a public-domain-by-copyright-expiration claim, for which SPDX has no direct term, use the CC Public Domain Mark IRI (`https://creativecommons.org/publicdomain/mark/1.0/`) as the current best-effort convention pending a more precise term. |
+| `creator` | yes | array (min 1) | One or more parties involved in producing this dataset — each a `schema:Organization` or `schema:Person` with a `name`, and an optional `role` (`originator`, `curator`, or `distributor`) to distinguish who made the underlying data from who aggregated or redistributed it. Omit `role` for a single, undifferentiated creator. |
+| `created` | yes | xsd:dateTime | When this provenance record was created. |
+| `origin` | yes | string | Free-text description of where the dataset's underlying data came from. |
+| `custodyChain` | yes | array (min 1) | Ordered chain of custody events, earliest first. Each entry: `agent` (IRI), `action` (string, e.g. `"collected"`, `"ingested"`, `"transformed"`), `startTime` (xsd:dateTime). A single-hop dataset has exactly one entry. |
+
+## Non-goals for 0.0.1
+
+- **No signature block.** Cryptographic signing is Sourcelume Attest's responsibility; this
+ version defines the record's shape, not how it gets signed.
+- **No adjudication fields.** There is no field for "verified: true/false" — Sourcelume publishes
+ claims, and verification is a downstream concern for registries/auditors, not the record itself.
+- **No usage/audit metadata.** `ProvenanceRecord` is deliberately dataset-centric. A model
+ producer's downstream usage claims about a dataset (e.g. a memorisation audit) belong to a
+ separate, later record type, not to this one — this is a scope decision worth defending on the
+ dev list rather than a settled fact, see `notes/stress-test-dfm-mimir.md`.
+
+## See also
+
+- [`context/0.0.1/sourcelume.jsonld`](../../context/0.0.1/sourcelume.jsonld) — the JSON-LD context.
+- [`schema/0.0.1/sourcelume.schema.json`](../../schema/0.0.1/sourcelume.schema.json) — structural validation.
+- [`schema/0.0.1/sourcelume.shacl.ttl`](../../schema/0.0.1/sourcelume.shacl.ttl) — RDF-level validation.
+- [`examples/minimal-record.jsonld`](../../examples/minimal-record.jsonld) — a conformant example.
+- `mappings/` — crosswalks to Croissant, the SPDX AI Profile, and OTDI (draft).
+- [`notes/stress-test-dfm-mimir.md`](../../notes/stress-test-dfm-mimir.md) — real-dataset stress
+ test that motivated the `creator`/`custodyChain` shape above and raises the still-open
+ usage/audit scope question.
diff --git a/tools/validate.py b/tools/validate.py
index 9528b65..6dedb92 100644
--- a/tools/validate.py
+++ b/tools/validate.py
@@ -1,4 +1,4 @@
-#
+#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -6,170 +6,77 @@
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
-"""Validate Sourcelume specification assets.
-
-This is a starting skeleton. It currently checks that the versioned
-schema, context, and SHACL files for each published version are present
-and syntactically valid (valid JSON / valid Turtle). It does not yet
-validate examples against the schema/SHACL, or check cross-version
-consistency — see SETUP.md for the full intended scope.
-Usage:
- uv run python tools/validate.py
-"""
-
-from __future__ import annotations
+"""Validates every examples/*.jsonld record against the JSON Schema and SHACL
+shapes for a spec version. Run via `uv run python tools/validate.py`."""
import json
-import re
import sys
from pathlib import Path
-VERSION_DIR_PATTERN = re.compile(r"^\d+\.\d+\.\d+$")
-
-REPO_ROOT = Path(__file__).resolve().parent.parent
-CONTEXT_DIR = REPO_ROOT / "context"
-SCHEMA_DIR = REPO_ROOT / "schema"
-EXAMPLES_DIR = REPO_ROOT / "examples"
-
-# Convert a filesystem path to a repository-relative string when possible.
-def display_path(path: Path) -> str:
- """Return `path` relative to the repository root when possible."""
- try:
- return str(path.relative_to(REPO_ROOT))
- except ValueError:
- return str(path)
-
-
-# Check that a required path exists and is a regular file.
-def require_file(path: Path) -> list[str]:
- """Return an error if `path` is not a regular file."""
- if not path.is_file():
- return [f"{display_path(path)}: missing required file"]
- return []
-
-
-# Find all published version directories whose names match the expected SemVer format.
-def find_version_dirs(base: Path) -> list[Path]:
- """Return published SemVer-like version directories under `base`."""
- if not base.is_dir():
- return []
- return sorted(
- p for p in base.iterdir()
- if p.is_dir() and not p.is_symlink() and VERSION_DIR_PATTERN.fullmatch(p.name)
- )
-
-
-# Validate that a file can be read and parsed as JSON.
-def check_json_file(path: Path) -> list[str]:
- errors = []
- try:
- json.loads(path.read_text(encoding="utf-8"))
- except json.JSONDecodeError as exc:
- errors.append(f"{display_path(path)}: invalid JSON ({exc})")
- except OSError as exc:
- errors.append(f"{display_path(path)}: could not read file ({exc})")
- return errors
-
-
-# Validate that a Turtle file can be read and parsed with rdflib.
-def check_turtle_file(path: Path) -> list[str]:
- errors = []
- try:
- import rdflib
- except ImportError as exc:
- errors.append(f"{display_path(path)}: cannot validate Turtle because rdflib is not installed ({exc})")
- return errors
-
- try:
- rdflib.Graph().parse(str(path), format="turtle")
- except OSError as exc:
- errors.append(f"{display_path(path)}: could not read file ({exc})")
- except Exception as exc: # noqa: BLE001 - report any parse failure
- errors.append(f"{display_path(path)}: invalid Turtle ({exc})")
- return errors
-
+from jsonschema import Draft202012Validator
+from pyshacl import validate as shacl_validate
+from rdflib import Graph
-# Validate required schema, context, and SHACL assets for each published version.
-def validate_schema_and_context() -> list[str]:
- errors: list[str] = []
+SPEC_VERSION = "0.0.1"
+BASE = Path(__file__).resolve().parent.parent
+SCHEMA_DIR = BASE / "schema" / SPEC_VERSION
+EXAMPLES_DIR = BASE / "examples"
- context_version_dirs = find_version_dirs(CONTEXT_DIR)
- schema_version_dirs = find_version_dirs(SCHEMA_DIR)
- context_versions = {version_dir.name for version_dir in context_version_dirs}
- schema_versions = {version_dir.name for version_dir in schema_version_dirs}
+def load_validator() -> Draft202012Validator:
+ schema = json.loads((SCHEMA_DIR / "sourcelume.schema.json").read_text())
+ return Draft202012Validator(schema, format_checker=Draft202012Validator.FORMAT_CHECKER)
- for version in sorted(context_versions - schema_versions):
- errors.append(
- f"{display_path(SCHEMA_DIR / version)}: missing schema directory for context version {version}"
- )
- for version in sorted(schema_versions - context_versions):
- errors.append(
- f"{display_path(CONTEXT_DIR / version)}: missing context directory for schema version {version}"
- )
+def validate_example(example: Path, validator: Draft202012Validator, shapes_path: Path) -> bool:
+ doc = json.loads(example.read_text())
- for version_dir in context_version_dirs:
- context_file = version_dir / "sourcelume.jsonld"
+ schema_errors = sorted(validator.iter_errors(doc), key=lambda e: list(e.path))
+ if schema_errors:
+ print(f"FAIL {example.name}: JSON Schema errors:")
+ for err in schema_errors:
+ print(f" - {list(err.path)}: {err.message}")
+ return False
- errors.extend(require_file(context_file))
- if context_file.is_file():
- errors.extend(check_json_file(context_file))
+ data_graph = Graph()
+ data_graph.parse(str(example), format="json-ld")
- for version_dir in schema_version_dirs:
- schema_file = version_dir / "sourcelume.schema.json"
- shacl_file = version_dir / "sourcelume.shacl.ttl"
-
- errors.extend(require_file(schema_file))
- errors.extend(require_file(shacl_file))
-
- if schema_file.is_file():
- errors.extend(check_json_file(schema_file))
- if shacl_file.is_file():
- errors.extend(check_turtle_file(shacl_file))
-
- return errors
-
-
-# Validate example JSON-LD files when an examples directory is present.
-def validate_examples() -> list[str]:
- errors: list[str] = []
- if not EXAMPLES_DIR.is_dir():
- return errors
-
- for example_file in EXAMPLES_DIR.rglob("*.jsonld"):
- if example_file.is_file() and not example_file.is_symlink():
- errors.extend(check_json_file(example_file))
-
- # TODO: validate each example against the corresponding version's
- # schema (jsonschema) and SHACL shapes (pyshacl), once example files
- # and a real data model exist.
+ conforms, _, results_text = shacl_validate(
+ data_graph,
+ shacl_graph=str(shapes_path),
+ shacl_graph_format="turtle",
+ inference="none",
+ )
+ if not conforms:
+ print(f"FAIL {example.name}: SHACL errors:\n{results_text}")
+ return False
- return errors
+ print(f"PASS {example.name}")
+ return True
-# Run all validation checks and return a process exit code.
def main() -> int:
- errors = validate_schema_and_context() + validate_examples()
-
- if errors:
- print("Validation failed:")
- for error in errors:
- print(f" - {error}")
+ examples = sorted(EXAMPLES_DIR.glob("*.jsonld"))
+ if not examples:
+ print(f"No *.jsonld files found under {EXAMPLES_DIR} — nothing to validate.", file=sys.stderr)
return 1
- print("Validation passed (skeleton checks only; see TODOs in this file).")
- return 0
+ validator = load_validator()
+ shapes_path = SCHEMA_DIR / "sourcelume.shacl.ttl"
+
+ ok = all(
+ [validate_example(example, validator, shapes_path) for example in examples]
+ )
+ return 0 if ok else 1
if __name__ == "__main__":
- sys.exit(main())
\ No newline at end of file
+ sys.exit(main())
diff --git a/uv.lock b/uv.lock
index 7898138..1a15db9 100644
--- a/uv.lock
+++ b/uv.lock
@@ -6,6 +6,19 @@ resolution-markers = [
"python_full_version < '3.12'",
]
+[[package]]
+name = "arrow"
+version = "1.4.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "python-dateutil" },
+ { name = "tzdata" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b9/33/032cdc44182491aa708d06a68b62434140d8c50820a087fac7af37703357/arrow-1.4.0.tar.gz", hash = "sha256:ed0cc050e98001b8779e84d461b0098c4ac597e88704a655582b21d116e526d7", size = 152931, upload-time = "2025-10-18T17:46:46.761Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl", hash = "sha256:749f0769958ebdc79c173ff0b0670d59051a535fa26e8eba02953dc19eb43205", size = 68797, upload-time = "2025-10-18T17:46:45.663Z" },
+]
+
[[package]]
name = "attrs"
version = "26.1.0"
@@ -24,6 +37,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/d7/3d/9487690d0e937854db587205c66bab3c3cf88d9f00ed380b74cb88cc94ee/cachetools-7.1.8-py3-none-any.whl", hash = "sha256:a81e3844acaa7355b6567f97bd67a94a14ec3a9bc2cbbdae45b9592cc036775b", size = 16842, upload-time = "2026-08-31T19:02:52.554Z" },
]
+[[package]]
+name = "fqdn"
+version = "1.5.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/30/3e/a80a8c077fd798951169626cde3e239adeba7dab75deb3555716415bd9b0/fqdn-1.5.1.tar.gz", hash = "sha256:105ed3677e767fb5ca086a0c1f4bb66ebc3c100be518f0e0d755d9eae164d89f", size = 6015, upload-time = "2021-03-11T07:16:29.08Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl", hash = "sha256:3a179af3761e4df6eb2e026ff9e1a3033d3587bf980a0b1b2e1e5d08d7358014", size = 9121, upload-time = "2021-03-11T07:16:28.351Z" },
+]
+
[[package]]
name = "frozendict"
version = "2.4.7"
@@ -42,18 +64,48 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/7d/c9/f6e1e8567660bc5b0aba281f2b0017b2a7665fcad6bf3ed67286a0c72cd4/html5rdf-1.2.1-py2.py3-none-any.whl", hash = "sha256:1f519121bc366af3e485310dc8041d2e86e5173c1a320fac3dc9d2604069b83e", size = 109765, upload-time = "2024-10-30T05:06:52.507Z" },
]
+[[package]]
+name = "idna"
+version = "3.19"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/5f/f7/abb373e5757eaec4b922b92f97ec8d6d7e057cf06778247604fbc4e7c3f3/idna-3.19.tar.gz", hash = "sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15", size = 215237, upload-time = "2026-08-18T05:14:24.27Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/57/b0/0e52c878c53f245edd3a11020f20979b3f490f245af532c7cae3027754b5/idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4", size = 68550, upload-time = "2026-08-18T05:14:22.343Z" },
+]
+
[[package]]
name = "importlib-metadata"
version = "9.0.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "zipp" },
+ { name = "zipp", marker = "python_full_version < '3.12'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/6f/7e/1e7e8dc30634b93ebb3d58a3dea569ad146e656218d3960ab04f62047b29/importlib_metadata-9.0.1.tar.gz", hash = "sha256:ab830580bc0ef3db61ce8fae716389e5462b67e033018bab6d8f80ef17172f99", size = 59124, upload-time = "2026-08-28T15:30:34.646Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/b3/55/ecca97ae19075f1fac62def77731e7f535e6c1fb8f92ff08160c5e6dade8/importlib_metadata-9.0.1-py3-none-any.whl", hash = "sha256:bba5600596a7e21f3eef53281cf28d6a5195634d2f2b78ff9501a3272c6eaab0", size = 27920, upload-time = "2026-08-28T15:30:33.433Z" },
]
+[[package]]
+name = "isoduration"
+version = "20.11.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "arrow" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/7c/1a/3c8edc664e06e6bd06cce40c6b22da5f1429aa4224d0c590f3be21c91ead/isoduration-20.11.0.tar.gz", hash = "sha256:ac2f9015137935279eac671f94f89eb00584f940f5dc49462a0c4ee692ba1bd9", size = 11649, upload-time = "2020-11-01T11:00:00.312Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/7b/55/e5326141505c5d5e34c5e0935d2908a74e4561eca44108fbfb9c13d2911a/isoduration-20.11.0-py3-none-any.whl", hash = "sha256:b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042", size = 11321, upload-time = "2020-11-01T10:59:58.02Z" },
+]
+
+[[package]]
+name = "jsonpointer"
+version = "3.1.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/18/c7/af399a2e7a67fd18d63c40c5e62d3af4e67b836a2107468b6a5ea24c4304/jsonpointer-3.1.1.tar.gz", hash = "sha256:0b801c7db33a904024f6004d526dcc53bbb8a4a0f4e32bfd10beadf60adf1900", size = 9068, upload-time = "2026-03-23T22:32:32.458Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/9e/6a/a83720e953b1682d2d109d3c2dbb0bc9bf28cc1cbc205be4ef4be5da709d/jsonpointer-3.1.1-py3-none-any.whl", hash = "sha256:8ff8b95779d071ba472cf5bc913028df06031797532f08a7d5b602d8b2a488ca", size = 7659, upload-time = "2026-03-23T22:32:31.568Z" },
+]
+
[[package]]
name = "jsonschema"
version = "4.26.0"
@@ -69,6 +121,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" },
]
+[package.optional-dependencies]
+format = [
+ { name = "fqdn" },
+ { name = "idna" },
+ { name = "isoduration" },
+ { name = "jsonpointer" },
+ { name = "rfc3339-validator" },
+ { name = "rfc3987" },
+ { name = "uri-template" },
+ { name = "webcolors" },
+]
+
[[package]]
name = "jsonschema-specifications"
version = "2025.9.1"
@@ -285,6 +349,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/03/90/7f35a79db93032ef20db5b740062b54afba32a2c2475a6f0a43c141a69de/pyshacl-0.40.1-py3-none-any.whl", hash = "sha256:27dd58c8ddfa103303b4a8c40b2c666332ffc912dbcd3137f7adc7b7bc5e6bda", size = 1306209, upload-time = "2026-07-28T01:37:34.298Z" },
]
+[[package]]
+name = "python-dateutil"
+version = "2.9.0.post0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "six" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" },
+]
+
[[package]]
name = "rdflib"
version = "7.6.0"
@@ -316,6 +392,27 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" },
]
+[[package]]
+name = "rfc3339-validator"
+version = "0.1.4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "six" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/28/ea/a9387748e2d111c3c2b275ba970b735e04e15cdb1eb30693b6b5708c4dbd/rfc3339_validator-0.1.4.tar.gz", hash = "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b", size = 5513, upload-time = "2021-05-12T16:37:54.178Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl", hash = "sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa", size = 3490, upload-time = "2021-05-12T16:37:52.536Z" },
+]
+
+[[package]]
+name = "rfc3987"
+version = "1.3.8"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/14/bb/f1395c4b62f251a1cb503ff884500ebd248eed593f41b469f89caa3547bd/rfc3987-1.3.8.tar.gz", hash = "sha256:d3c4d257a560d544e9826b38bc81db676890c79ab9d7ac92b39c7a253d5ca733", size = 20700, upload-time = "2018-07-29T17:23:47.954Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/65/d4/f7407c3d15d5ac779c3dd34fbbc6ea2090f77bd7dd12f207ccf881551208/rfc3987-1.3.8-py2.py3-none-any.whl", hash = "sha256:10702b1e51e5658843460b189b185c0366d2cf4cff716f13111b0ea9fd2dce53", size = 13377, upload-time = "2018-07-29T17:23:45.313Z" },
+]
+
[[package]]
name = "rpds-py"
version = "2026.6.3"
@@ -439,6 +536,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/10/85/48f0abdcef5cce4e034c7a5b0ceeceba0b01bf0d942824f4bb720afe2dec/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8e65860d238379ed982fd9ba690579b5e95af2f4840f99c772816dbe573cb826", size = 586486, upload-time = "2026-06-30T07:17:51.141Z" },
]
+[[package]]
+name = "six"
+version = "1.17.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" },
+]
+
[[package]]
name = "sourcelume-spec"
version = "0.0.1"
@@ -446,7 +552,7 @@ source = { virtual = "." }
[package.optional-dependencies]
dev = [
- { name = "jsonschema" },
+ { name = "jsonschema", extra = ["format"] },
{ name = "pyld" },
{ name = "pyshacl" },
{ name = "rdflib" },
@@ -454,7 +560,7 @@ dev = [
[package.metadata]
requires-dist = [
- { name = "jsonschema", marker = "extra == 'dev'" },
+ { name = "jsonschema", extras = ["format"], marker = "extra == 'dev'" },
{ name = "pyld", marker = "extra == 'dev'" },
{ name = "pyshacl", marker = "extra == 'dev'" },
{ name = "rdflib", marker = "extra == 'dev'" },
@@ -470,6 +576,24 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" },
]
+[[package]]
+name = "tzdata"
+version = "2026.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" },
+]
+
+[[package]]
+name = "uri-template"
+version = "1.3.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/31/c7/0336f2bd0bcbada6ccef7aaa25e443c118a704f828a0620c6fa0207c1b64/uri-template-1.3.0.tar.gz", hash = "sha256:0e00f8eb65e18c7de20d595a14336e9f337ead580c70934141624b6d1ffdacc7", size = 21678, upload-time = "2023-06-21T01:49:05.374Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl", hash = "sha256:a44a133ea12d44a0c0f06d7d42a52d71282e77e2f937d8abd5655b8d56fc1363", size = 11140, upload-time = "2023-06-21T01:49:03.467Z" },
+]
+
[[package]]
name = "wcwidth"
version = "0.8.3"
@@ -479,6 +603,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/c4/0e/57f6bb3024a597b2e8ec4aee710ffe62ddc95af2e2bb1ee7a7abdc22c68c/wcwidth-0.8.3-py3-none-any.whl", hash = "sha256:d5b73dba6158a595ec9370350e7f2637bcac8d6c5e4fde34f30fcffb6103a5e4", size = 331669, upload-time = "2026-08-28T18:10:04.909Z" },
]
+[[package]]
+name = "webcolors"
+version = "25.10.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/1d/7a/eb316761ec35664ea5174709a68bbd3389de60d4a1ebab8808bfc264ed67/webcolors-25.10.0.tar.gz", hash = "sha256:62abae86504f66d0f6364c2a8520de4a0c47b80c03fc3a5f1815fedbef7c19bf", size = 53491, upload-time = "2025-10-31T07:51:03.977Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl", hash = "sha256:032c727334856fc0b968f63daa252a1ac93d33db2f5267756623c210e57a4f1d", size = 14905, upload-time = "2025-10-31T07:51:01.778Z" },
+]
+
[[package]]
name = "zipp"
version = "4.1.0"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@sourcelume.apache.org
For additional commands, e-mail: commits-help@sourcelume.apache.org