You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hamilton.apache.org by zi...@apache.org on 2025/09/06 00:20:14 UTC
(hamilton) branch feat/hamilton-core updated (364e16a2 -> 399b4a70)
This is an automated email from the ASF dual-hosted git repository.
zilto pushed a change to branch feat/hamilton-core
in repository https://gitbox.apache.org/repos/asf/hamilton.git
discard 364e16a2 setup.py now copies the main source code
discard c4987ea2 implement module proxying logic
discard b55e97ee dynamically define package using setup.py from pyproject.toml
discard e6fe1937 add README explanations; wip
discard 0b50e607 use absolute import as fallback; required by proxy
discard 009b5823 use lazy imports for pandas and numpy
add 875555f7 Clean up repo for first ASF release (Attempt 2) (#1374)
new 3c66fc4d use lazy imports for pandas and numpy
new 588638ce use absolute import as fallback; required by proxy
new c1deb440 add README explanations; wip
new c63886b7 dynamically define package using setup.py from pyproject.toml
new 543d2672 implement module proxying logic
new 399b4a70 setup.py now copies the main source code
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (364e16a2)
\
N -- N -- N refs/heads/feat/hamilton-core (399b4a70)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.flake8 | 20 ----
.github/workflows/hamilton-main.yml | 26 ++---
.github/workflows/sphinx-docs.yml | 2 +-
.pre-commit-config.yaml | 4 -
.style.yapf | 23 ----
MANIFEST.in | 2 -
dag.png => assets/dag.png | Bin
assets/dag_example_module.png | Bin 0 -> 27223 bytes
hamiltondag.png => assets/hamiltondag.png | Bin
.../hello_world_image.png | Bin
dag_example_module.png | Bin 27367 -> 0 bytes
README-DOCS.md => docs/README.md | 2 +-
docs/conf.py | 6 +
pyproject.toml | 121 +++++++++++----------
build_conda.sh => scripts/build_conda.sh | 0
setup.py | 49 ---------
tests/plugins/test_polars_extensions.py | 9 +-
basics.md => writeups/basics.md | 0
data_quality.md => writeups/data_quality.md | 0
decorators.md => writeups/decorators.md | 0
developer_setup.md => writeups/developer_setup.md | 4 +-
21 files changed, 92 insertions(+), 176 deletions(-)
delete mode 100644 .flake8
delete mode 100644 .style.yapf
rename dag.png => assets/dag.png (100%)
create mode 100644 assets/dag_example_module.png
rename hamiltondag.png => assets/hamiltondag.png (100%)
rename hello_world_image.png => assets/hello_world_image.png (100%)
delete mode 100644 dag_example_module.png
rename README-DOCS.md => docs/README.md (97%)
rename build_conda.sh => scripts/build_conda.sh (100%)
delete mode 100644 setup.py
rename basics.md => writeups/basics.md (100%)
rename data_quality.md => writeups/data_quality.md (100%)
rename decorators.md => writeups/decorators.md (100%)
rename developer_setup.md => writeups/developer_setup.md (97%)
(hamilton) 01/06: use lazy imports for pandas and numpy
Posted by zi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zilto pushed a commit to branch feat/hamilton-core
in repository https://gitbox.apache.org/repos/asf/hamilton.git
commit 3c66fc4d8efe9f9c481fe009853fb27aecd96615
Author: zilto <68...@users.noreply.github.com>
AuthorDate: Tue Sep 2 21:20:17 2025 -0400
use lazy imports for pandas and numpy
---
hamilton/base.py | 21 +++++++++++++++++----
1 file changed, 17 insertions(+), 4 deletions(-)
diff --git a/hamilton/base.py b/hamilton/base.py
index 02241aac..4fa5050c 100644
--- a/hamilton/base.py
+++ b/hamilton/base.py
@@ -19,15 +19,13 @@
It should only import hamilton.node, numpy, pandas.
It cannot import hamilton.graph, or hamilton.driver.
"""
+from __future__ import annotations
import abc
import collections
import logging
-from typing import Any, Dict, List, Optional, Tuple, Type, Union
+from typing import Any, Dict, List, Optional, Tuple, Type, Union, TYPE_CHECKING
-import numpy as np
-import pandas as pd
-from pandas.core.indexes import extension as pd_extension
from hamilton.lifecycle import api as lifecycle_api
@@ -36,6 +34,11 @@ try:
except ImportError:
import node
+if TYPE_CHECKING:
+ import numpy as np
+ import pandas as pd
+
+
logger = logging.getLogger(__name__)
@@ -120,6 +123,8 @@ class PandasDataFrameResult(ResultMixin):
:param outputs: the dict we're trying to create a result from.
:return: dict of all index types, dict of time series/categorical index types, dict if there is no index
"""
+ import pandas as pd
+
all_index_types = collections.defaultdict(list)
time_indexes = collections.defaultdict(list)
no_indexes = collections.defaultdict(list)
@@ -131,6 +136,8 @@ class PandasDataFrameResult(ResultMixin):
def get_parent_time_index_type():
"""Helper to pull the right time index parent class."""
+ from pandas.core.indexes import extension as pd_extension
+
if hasattr(pd_extension, "NDArrayBackedExtensionIndex"):
index_type = pd_extension.NDArrayBackedExtensionIndex
else:
@@ -220,6 +227,8 @@ class PandasDataFrameResult(ResultMixin):
:param outputs: the outputs to build a dataframe from.
"""
+ import pandas as pd
+
# TODO check inputs are pd.Series, arrays, or scalars -- else error
output_index_type_tuple = PandasDataFrameResult.pandas_index_types(outputs)
# this next line just log warnings
@@ -255,6 +264,7 @@ class PandasDataFrameResult(ResultMixin):
:param outputs: The outputs to build the dataframe from.
:return: A dataframe with the outputs.
"""
+ import pandas as pd
def get_output_name(output_name: str, column_name: str) -> str:
"""Add function prefix to columns.
@@ -300,6 +310,7 @@ class PandasDataFrameResult(ResultMixin):
return [Any]
def output_type(self) -> Type:
+ import pandas as pd
return pd.DataFrame
@@ -365,6 +376,7 @@ class NumpyMatrixResult(ResultMixin):
:param outputs: function_name -> np.array.
:return: numpy matrix
"""
+ import numpy as np
# TODO check inputs are all numpy arrays/array like things -- else error
num_rows = -1
columns_with_lengths = collections.OrderedDict()
@@ -402,6 +414,7 @@ class NumpyMatrixResult(ResultMixin):
return [Any] # Typing
def output_type(self) -> Type:
+ import pandas as pd
return pd.DataFrame
(hamilton) 05/06: implement module proxying logic
Posted by zi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zilto pushed a commit to branch feat/hamilton-core
in repository https://gitbox.apache.org/repos/asf/hamilton.git
commit 543d2672aaac33a455a5ea712c98319dddce7b40
Author: zilto <68...@users.noreply.github.com>
AuthorDate: Tue Sep 2 21:39:22 2025 -0400
implement module proxying logic
---
hamilton-core/hamilton/__init__.py | 68 ++++++++++++++++++++++++++++++++++++++
1 file changed, 68 insertions(+)
diff --git a/hamilton-core/hamilton/__init__.py b/hamilton-core/hamilton/__init__.py
new file mode 100644
index 00000000..3fdd2f4a
--- /dev/null
+++ b/hamilton-core/hamilton/__init__.py
@@ -0,0 +1,68 @@
+import importlib.util
+import pathlib
+import sys
+from typing import Any
+from types import ModuleType
+
+
+def _load_hamilton_module() -> ModuleType:
+ """Patch this relative import in the Hamilton core repository
+
+ ```python
+ # hamilton/__init__.py
+ try:
+ from .version import VERSION as __version__ # noqa: F401
+ except ImportError:
+ from version import VERSION as __version__ # noqa: F401
+ ```
+ """
+
+ origin_path = pathlib.Path(__file__).parent / "_hamilton" / "__init__.py"
+ origin_spec = importlib.util.spec_from_file_location("hamilton", origin_path)
+ origin_module = importlib.util.module_from_spec(origin_spec)
+
+ # The following lines are only required if we don't modify `hamilton/__init__.py`
+ # source_segment = "from version import VERSION as __version__"
+ # # the namespace `hamilton._hamilton` is only temporarily available; it will be removed
+ # # by the end of this initialization
+ # patched_segment = "from hamilton._hamilton.version import VERSION as __version__"
+
+ # source_code = pathlib.Path(origin_path).read_text()
+ # patched_code = source_code.replace(source_segment, patched_segment)
+
+ # exec(patched_code, origin_module.__dict__)
+ # sys.modules["hamilton"] = origin_module
+
+ origin_spec.loader.exec_module(origin_module)
+ return origin_module
+
+
+def _load_hamilton_registry_module():
+ module_path = pathlib.Path(__file__).parent / "_hamilton" / "registry.py"
+ module_spec = importlib.util.spec_from_file_location("hamilton.registry", module_path)
+ module = importlib.util.module_from_spec(module_spec)
+ module_spec.loader.exec_module(module)
+ return module
+
+
+def _create_proxy_module() -> ModuleType:
+ proxy_module = ModuleType(__name__)
+ sys.modules[__name__] = proxy_module
+ return proxy_module
+
+
+_registry_module = _load_hamilton_registry_module()
+# disable plugin autoloading
+_registry_module.disable_autoload()
+
+_origin_module = _load_hamilton_module()
+_proxy_module = _create_proxy_module()
+
+def __getattr__(name: str) -> Any:
+ try:
+ return getattr(_origin_module, name)
+ except AttributeError:
+ raise AttributeError(f"module {__name__} has no attribute {name}")
+
+# `getattr()` must be available to build the package
+_proxy_module.__getattr__ = __getattr__
(hamilton) 06/06: setup.py now copies the main source code
Posted by zi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zilto pushed a commit to branch feat/hamilton-core
in repository https://gitbox.apache.org/repos/asf/hamilton.git
commit 399b4a7020919089fac603608263383b2f13e960
Author: zilto <68...@users.noreply.github.com>
AuthorDate: Tue Sep 2 21:57:10 2025 -0400
setup.py now copies the main source code
---
hamilton-core/.gitignore | 1 +
hamilton-core/setup.py | 32 ++++++++++++++++++++++++++++++++
2 files changed, 33 insertions(+)
diff --git a/hamilton-core/.gitignore b/hamilton-core/.gitignore
new file mode 100644
index 00000000..750d8503
--- /dev/null
+++ b/hamilton-core/.gitignore
@@ -0,0 +1 @@
+hamilton/_hamilton
diff --git a/hamilton-core/setup.py b/hamilton-core/setup.py
index da780c52..0a4dfa05 100644
--- a/hamilton-core/setup.py
+++ b/hamilton-core/setup.py
@@ -2,8 +2,38 @@
import tomllib
import pathlib
import re
+import os
+import shutil
+import sys
from setuptools import setup
+# ensure the right current working directory
+os.chdir(os.path.abspath(os.path.dirname(__file__)))
+
+def copy_hamilton_library():
+ setup_dir = pathlib.Path(__file__).resolve().parent
+ source_dir = (setup_dir.parent / 'hamilton').resolve()
+ dest_dir = (setup_dir / 'hamilton' / '_hamilton').resolve()
+
+ # Safety checks
+ if not source_dir.is_dir():
+ print(f"Error: Source directory does not exist: {source_dir}")
+ sys.exit(1)
+
+ if not str(dest_dir).startswith(str(setup_dir)):
+ print(f"Error: Destination directory {dest_dir} is outside the setup directory {setup_dir}")
+ sys.exit(1)
+
+ # Remove destination if it exists to avoid errors or stale files
+ if dest_dir.exists():
+ print("delete: ", dest_dir)
+ shutil.rmtree(dest_dir)
+
+ # Copy entire directory tree from source to destination
+ print(f"copy from: {source_dir}; to {dest_dir}")
+ shutil.copytree(source_dir, dest_dir)
+
+
def get_version():
version_path = pathlib.Path(__file__).parent / "hamilton" / "_hamilton" / "version.py"
content = version_path.read_text()
@@ -15,6 +45,8 @@ def get_version():
version_str = ".".join(version_parts)
return version_str
+copy_hamilton_library()
+
pyproject_path = pathlib.Path(__file__).parents[1] / "pyproject.toml"
pyproject = tomllib.loads(pyproject_path.read_text())
project = pyproject["project"]
(hamilton) 02/06: use absolute import as fallback; required by proxy
Posted by zi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zilto pushed a commit to branch feat/hamilton-core
in repository https://gitbox.apache.org/repos/asf/hamilton.git
commit 588638ce91f7dc0dff7b7935e41a1d911e3c05ec
Author: zilto <68...@users.noreply.github.com>
AuthorDate: Tue Sep 2 21:20:58 2025 -0400
use absolute import as fallback; required by proxy
---
hamilton/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/hamilton/__init__.py b/hamilton/__init__.py
index a302407d..4d3fd39f 100644
--- a/hamilton/__init__.py
+++ b/hamilton/__init__.py
@@ -1,7 +1,7 @@
try:
from .version import VERSION as __version__ # noqa: F401
except ImportError:
- from version import VERSION as __version__ # noqa: F401
+ from hamilton.version import VERSION as __version__ # noqa: F401
# this supposedly is required for namespace packages to work.
__path__ = __import__("pkgutil").extend_path(__path__, __name__)
(hamilton) 03/06: add README explanations; wip
Posted by zi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zilto pushed a commit to branch feat/hamilton-core
in repository https://gitbox.apache.org/repos/asf/hamilton.git
commit c1deb44066074182e5d0dac0e237e3eec9093217
Author: zilto <68...@users.noreply.github.com>
AuthorDate: Tue Sep 2 21:38:25 2025 -0400
add README explanations; wip
---
hamilton-core/README.md | 41 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 41 insertions(+)
diff --git a/hamilton-core/README.md b/hamilton-core/README.md
new file mode 100644
index 00000000..52accb30
--- /dev/null
+++ b/hamilton-core/README.md
@@ -0,0 +1,41 @@
+# Read carefully
+
+> Use at your own risk
+
+This directory contains code for the package `sf-hamilton-core`. It is a drop-in replacement of `sf-hamilton`, with two changes:
+- disable plugin autoloading
+- make `pandas` and `numpy` optional dependencies; and remove `networkx` dependency (currently unused).
+
+This makes the Hamilton package a much lighter install and solves long library loading time.
+
+## As a user
+If you want to try `sf-hamilton-core`, you need to:
+1. Remove your current Hamilton installation: `pip uninstall sf-hamilton`
+2. Install Hamilton core `pip install sf-hamilton-core`
+3. Check installation `pip list` should only include `sf-hamilton-core`.
+
+This will install a different Python package with the name `hamilton` with the smaller dependencies and plugin autoloading disabled.
+
+It should be a drop-in replacement and your existing Hamilton code should just work. Though, if you're relying on plugins (e.g., parquet materializers, dataframe result builders), you will need to manually load them.
+
+
+## How does it work
+
+
+## Why is another package `sf-hamilton` necessary
+This exists to prevent backwards incompatible changes for people who `pip install sf-hamilton` and use it in production. It is a temporary solution until a major release `sf-hamilton==2.0.0` could allow breaking changes and a more robust solution.
+
+### Disable plugin autoloading
+Hamilton has generous number of plugins (`pandas`, `polars`, `mlflow`, `spark`). To give a good user experience, Hamilton autoloads plugins based on the available Python libraries in the current Python environment. For example, `to.mlflow()` becomes available if `mlflow` is installed. Autoloaded features notably include materializers like `from_.parquet` and `to.parquet` and data validators (pydantic, pandera, etc.)
+
+The issue with this approach is that Python environment with a lot of dependencies, common in data science, can be very slow to start because of all the imports. Currently, Hamilton allows to disable autoloading via a user config or Python code. This require manual setups and is not the best default for some users.
+
+### `pandas` and `numpy` dependencies
+Hamilton was initially created for workflows that used `pandas` and `numpy` heavily. For this reason, `numpy` and `pandas` are imported at the top-level of module `hamilton.base`. Because of the package structure, as a Hamilton user, you're importing `pandas` and `numpy` every time you import `hamilton`.
+
+A reasonable change would be to move `numpy` and `pandas` to a "lazy" location. Then, dependencies would only be imported when features requiring them are used and they could be removed from `pyproject.toml`. Unfortunately, plugin autoloading defaults make this solution a significant breaking change and insatisfactory.
+
+Since plugins are loaded based on the Python package available, removing `pandas` and `numpy` would allow disable the loading of these plugins. This would break popular CSV and parquet materializers.
+
+### `networkx` dependency
+The `sf-hamilton[visualization]` extra currently includes `networkx` as a dependency, though it is never actually used. There's a single function requiring it and it could be implemented in pure Python. This has been made even easier with the addition of `graphlib` in the standard library in Python 3.9.
(hamilton) 04/06: dynamically define package using setup.py from pyproject.toml
Posted by zi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zilto pushed a commit to branch feat/hamilton-core
in repository https://gitbox.apache.org/repos/asf/hamilton.git
commit c63886b7a7da6f56f7b16a071c10e85f4a8e6803
Author: zilto <68...@users.noreply.github.com>
AuthorDate: Tue Sep 2 21:38:52 2025 -0400
dynamically define package using setup.py from pyproject.toml
---
hamilton-core/setup.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 52 insertions(+)
diff --git a/hamilton-core/setup.py b/hamilton-core/setup.py
new file mode 100644
index 00000000..da780c52
--- /dev/null
+++ b/hamilton-core/setup.py
@@ -0,0 +1,52 @@
+
+import tomllib
+import pathlib
+import re
+from setuptools import setup
+
+def get_version():
+ version_path = pathlib.Path(__file__).parent / "hamilton" / "_hamilton" / "version.py"
+ content = version_path.read_text()
+ match = re.search(r'^VERSION\s*=\s*\(([^)]+)\)', content, re.MULTILINE)
+ if match:
+ version_tuple_str = match.group(1) # "1, 88, 0"
+ # Parse tuple string into list of integers
+ version_parts = [part.strip() for part in version_tuple_str.split(",")]
+ version_str = ".".join(version_parts)
+ return version_str
+
+pyproject_path = pathlib.Path(__file__).parents[1] / "pyproject.toml"
+pyproject = tomllib.loads(pyproject_path.read_text())
+project = pyproject["project"]
+
+readme_file = project.get("readme", None)
+console_scripts = [
+ f"{name}={target}" for name, target in project.get("entry-points", {}).get("console_scripts", {}).items()
+]
+install_requires = list(
+ set(project.get("dependencies", [])).difference(set(["pandas", "numpy"]))
+)
+extras_require = {
+ **project.get("optional-dependencies", {}),
+ **{"visualization": ["graphviz"]}, # drop networkx
+}
+
+setup(
+ name="sf-hamilton-core",
+ version=get_version(),
+ description=project.get("description", ""),
+ long_description=pathlib.Path(readme_file).read_text() if readme_file else "",
+ long_description_content_type="text/markdown" if readme_file else None,
+ python_requires=project.get("requires-python", None),
+ license=project.get("license", {}).get("text", None),
+ keywords=project.get("keywords", []),
+ author=", ".join(a["name"] for a in project.get("authors", [])),
+ author_email=", ".join(a["email"] for a in project.get("authors", [])),
+ classifiers=project.get("classifiers", []),
+ install_requires=install_requires,
+ extras_require=extras_require,
+ entry_points={"console_scripts": console_scripts},
+ project_urls=project.get("urls", {}),
+ packages=["hamilton"],
+ package_data={"hamilton": ["*.json", "*.md", "*.txt"]},
+)