You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by pi...@apache.org on 2026/09/10 11:17:13 UTC
(iggy) branch master updated: feat(python): expose partition management (#4017)
This is an automated email from the ASF dual-hosted git repository.
spetz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/master by this push:
new df4c75d34 feat(python): expose partition management (#4017)
df4c75d34 is described below
commit df4c75d34614865ea3bc1ffb2b117c8247146091
Author: Elioooon <10...@users.noreply.github.com>
AuthorDate: Thu Sep 10 19:17:08 2026 +0800
feat(python): expose partition management (#4017)
Expose `create_partitions` and `delete_partitions` on the Python
`IggyClient`, delegating to the existing Rust SDK operations and error
conversion. Document ID reuse, validation limits, asynchronous
materialization, consumer-group rebalancing, and the permission
hierarchy. Creating a topic accepts zero partitions; adding or deleting
partitions requires a positive count.
Closes #4014
---
core/simulator/src/lib.rs | 20 +++
foreign/python/apache_iggy.pyi | 62 ++++++++-
foreign/python/src/client.rs | 203 ++++++++++++++++++-----------
foreign/python/tests/test_partition.py | 228 +++++++++++++++++++++++++++++++++
4 files changed, 439 insertions(+), 74 deletions(-)
diff --git a/core/simulator/src/lib.rs b/core/simulator/src/lib.rs
index 51a2d3559..3fd111c55 100644
--- a/core/simulator/src/lib.rs
+++ b/core/simulator/src/lib.rs
@@ -6305,6 +6305,17 @@ mod partition_repair_driver_tests {
below would have nothing to lose"
);
let counted = gap_drops(&sim, LAGGING, namespace);
+ let partition_stats = sim.replicas[LAGGING as usize]
+ .partition_shard(namespace)
+ .plane
+ .partitions()
+ .get_by_ns(&namespace)
+ .expect("partition exists before ConfirmRemove")
+ .stats
+ .clone();
+ let topic_stats = partition_stats.parent();
+ let stream_stats = topic_stats.parent();
+ assert!(partition_stats.messages_count_inconsistent() > 0);
// The reconciler's teardown order: the tombstone lands first and the
// disk delete runs before `ConfirmRemove`, so from here `get_mut_by_ns`
@@ -6327,6 +6338,15 @@ mod partition_repair_driver_tests {
"the {buffered} prepare(s) buffered on the partition went to the floor \
with it; the drops are the only record those frames existed"
);
+ assert_eq!(partition_stats.messages_count_inconsistent(), 0);
+ assert_eq!(partition_stats.size_bytes_inconsistent(), 0);
+ assert_eq!(partition_stats.segments_count_inconsistent(), 0);
+ assert_eq!(topic_stats.messages_count_inconsistent(), 0);
+ assert_eq!(topic_stats.size_bytes_inconsistent(), 0);
+ assert_eq!(topic_stats.segments_count_inconsistent(), 0);
+ assert_eq!(stream_stats.messages_count_inconsistent(), 0);
+ assert_eq!(stream_stats.size_bytes_inconsistent(), 0);
+ assert_eq!(stream_stats.segments_count_inconsistent(), 0);
}
}
diff --git a/foreign/python/apache_iggy.pyi b/foreign/python/apache_iggy.pyi
index 901af64aa..adebabee9 100644
--- a/foreign/python/apache_iggy.pyi
+++ b/foreign/python/apache_iggy.pyi
@@ -1325,7 +1325,7 @@ class IggyClient:
Args:
stream: Stream identifier as `str | int`.
name: Topic name as `str`.
- partitions_count: Number of partitions as `int`.
+ partitions_count: Number of partitions as `int`, at most 1000.
compression_algorithm: Compression algorithm as `str | None`.
message_expiry: Message expiry as `IggyExpiry | None`.
max_topic_size: Maximum topic size as `MaxTopicSize | None`.
@@ -1443,6 +1443,66 @@ class IggyClient:
Raises:
RuntimeError: If an identifier is invalid or the request fails.
"""
+ def create_partitions(
+ self,
+ stream_id: builtins.str | builtins.int,
+ topic_id: builtins.str | builtins.int,
+ partitions_count: builtins.int,
+ ) -> collections.abc.Awaitable[None]:
+ r"""
+ Create partitions for a topic. New partition IDs continue from one past the
+ current highest ID; IDs removed by deletion can be reused. Existing consumer
+ groups are immediately rebalanced across all partitions, advancing their
+ generation and dropping pending revocations.
+
+ Args:
+ stream_id: Stream identifier as `str | int`.
+ topic_id: Topic identifier as `str | int`.
+ partitions_count: Number of partitions to create as `int`, between 1 and
+ 1000 inclusive.
+
+ Returns:
+ An awaitable that resolves to `None` when the partitions are committed;
+ storage materialization completes asynchronously.
+
+ Raises:
+ ValueError: If an identifier is invalid.
+ OverflowError: If `partitions_count` is outside the unsigned 32-bit range.
+ RuntimeError: If the client is not authenticated, lacks global
+ `manage_streams` or `manage_topics`, per-stream `manage_stream` or
+ `manage_topics`, or per-topic `manage_topic` permission, or the
+ request fails.
+ """
+ def delete_partitions(
+ self,
+ stream_id: builtins.str | builtins.int,
+ topic_id: builtins.str | builtins.int,
+ partitions_count: builtins.int,
+ ) -> collections.abc.Awaitable[None]:
+ r"""
+ Delete the last partitions from a topic, including all messages stored in them.
+ Existing consumer groups are immediately rebalanced across the remaining
+ partitions, advancing their generation and dropping pending revocations.
+
+ Args:
+ stream_id: Stream identifier as `str | int`.
+ topic_id: Topic identifier as `str | int`.
+ partitions_count: Number of partitions to delete as `int` from the end of
+ the topic; must be between 1 and 1000 inclusive and no greater than
+ its current count.
+
+ Returns:
+ An awaitable that resolves to `None` when deletion is accepted; storage
+ teardown completes asynchronously.
+
+ Raises:
+ ValueError: If an identifier is invalid.
+ OverflowError: If `partitions_count` is outside the unsigned 32-bit range.
+ RuntimeError: If the client is not authenticated, lacks global
+ `manage_streams` or `manage_topics`, per-stream `manage_stream` or
+ `manage_topics`, or per-topic `manage_topic` permission, or the
+ request fails.
+ """
def create_consumer_group(
self,
stream_id: builtins.str | builtins.int,
diff --git a/foreign/python/src/client.rs b/foreign/python/src/client.rs
index 0fdef39d9..0e3a7b937 100644
--- a/foreign/python/src/client.rs
+++ b/foreign/python/src/client.rs
@@ -60,7 +60,7 @@ pub struct IggyClient {
inner: Arc<RustIggyClient>,
}
-/// Keeps the SDK's own message on the `RuntimeError` the Python surface raises.
+/// Converts SDK errors to the RuntimeError exposed by the Python API.
fn to_runtime_error<E: Display>(error: E) -> PyErr {
PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(error.to_string())
}
@@ -174,8 +174,8 @@ impl IggyClient {
// is a no-op for the other transports since the protocol isn't known until the
// connection string is parsed.
let _guard = pyo3_async_runtimes::tokio::get_runtime().enter();
- let client = RustIggyClient::from_connection_string(&connection_string)
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ let client =
+ RustIggyClient::from_connection_string(&connection_string).map_err(to_runtime_error)?;
Ok(Self {
inner: Arc::new(client),
})
@@ -186,12 +186,10 @@ impl IggyClient {
#[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))]
fn ping<'a>(&self, py: Python<'a>) -> PyResult<Bound<'a, PyAny>> {
let inner = self.inner.clone();
- future_into_py(py, async move {
- inner
- .ping()
- .await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))
- })
+ future_into_py(
+ py,
+ async move { inner.ping().await.map_err(to_runtime_error) },
+ )
}
/// Get the statistics and details of the server and its running process.
@@ -210,10 +208,7 @@ impl IggyClient {
fn get_stats<'a>(&self, py: Python<'a>) -> PyResult<Bound<'a, PyAny>> {
let inner = self.inner.clone();
future_into_py(py, async move {
- let stats = inner
- .get_stats()
- .await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ let stats = inner.get_stats().await.map_err(to_runtime_error)?;
Ok(PyStats::from(stats))
})
}
@@ -243,7 +238,7 @@ impl IggyClient {
let specs = inner
.describe_options(scope)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(specs
.into_iter()
.map(PyOptionSpec::from)
@@ -265,7 +260,7 @@ impl IggyClient {
inner
.login_user(&username, &password)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(())
})
}
@@ -288,10 +283,7 @@ impl IggyClient {
let inner = self.inner.clone();
future_into_py(py, async move {
- let user = inner
- .get_user(&user_id)
- .await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ let user = inner.get_user(&user_id).await.map_err(to_runtime_error)?;
Ok(user.map(PyUserInfoDetails::from))
})
}
@@ -308,10 +300,7 @@ impl IggyClient {
let inner = self.inner.clone();
future_into_py(py, async move {
- let users = inner
- .get_users()
- .await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ let users = inner.get_users().await.map_err(to_runtime_error)?;
Ok(users.into_iter().map(PyUserInfo::from).collect::<Vec<_>>())
})
}
@@ -349,7 +338,7 @@ impl IggyClient {
let user = inner
.create_user(&username, &password, status, permissions)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(PyUserInfoDetails::from(user))
})
}
@@ -390,7 +379,7 @@ impl IggyClient {
&UserUpdateOptions::default(),
)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(())
})
}
@@ -415,7 +404,7 @@ impl IggyClient {
inner
.delete_user(&user_id)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(())
})
}
@@ -453,7 +442,7 @@ impl IggyClient {
inner
.update_permissions(&user_id, permissions)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(())
})
}
@@ -486,7 +475,7 @@ impl IggyClient {
inner
.change_password(&user_id, ¤t_password, &new_password)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(())
})
}
@@ -503,10 +492,7 @@ impl IggyClient {
let inner = self.inner.clone();
future_into_py(py, async move {
- inner
- .logout_user()
- .await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ inner.logout_user().await.map_err(to_runtime_error)?;
Ok(())
})
}
@@ -519,10 +505,7 @@ impl IggyClient {
fn connect<'a>(&self, py: Python<'a>) -> PyResult<Bound<'a, PyAny>> {
let inner = self.inner.clone();
future_into_py(py, async move {
- inner
- .connect()
- .await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ inner.connect().await.map_err(to_runtime_error)?;
Ok(())
})
}
@@ -534,10 +517,7 @@ impl IggyClient {
fn create_stream<'a>(&self, py: Python<'a>, name: String) -> PyResult<Bound<'a, PyAny>> {
let inner = self.inner.clone();
future_into_py(py, async move {
- inner
- .create_stream(&name)
- .await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ inner.create_stream(&name).await.map_err(to_runtime_error)?;
Ok(())
})
}
@@ -558,7 +538,7 @@ impl IggyClient {
let stream = inner
.get_stream(&stream_id)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(stream.map(StreamDetails::from))
})
}
@@ -575,10 +555,7 @@ impl IggyClient {
fn get_streams<'a>(&self, py: Python<'a>) -> PyResult<Bound<'a, PyAny>> {
let inner = self.inner.clone();
future_into_py(py, async move {
- let streams = inner
- .get_streams()
- .await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ let streams = inner.get_streams().await.map_err(to_runtime_error)?;
Ok(streams.into_iter().map(Stream::from).collect::<Vec<_>>())
})
}
@@ -627,7 +604,7 @@ impl IggyClient {
inner
.update_stream(&stream_id, &name, &update_options)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(())
})
}
@@ -660,7 +637,7 @@ impl IggyClient {
inner
.delete_stream(&stream_id)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(())
})
}
@@ -694,7 +671,7 @@ impl IggyClient {
inner
.purge_stream(&stream_id)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(())
})
}
@@ -704,7 +681,7 @@ impl IggyClient {
/// Args:
/// stream: Stream identifier as `str | int`.
/// name: Topic name as `str`.
- /// partitions_count: Number of partitions as `int`.
+ /// partitions_count: Number of partitions as `int`, at most 1000.
/// compression_algorithm: Compression algorithm as `str | None`.
/// message_expiry: Message expiry as `IggyExpiry | None`.
/// max_topic_size: Maximum topic size as `MaxTopicSize | None`.
@@ -784,7 +761,7 @@ impl IggyClient {
inner
.create_topic(&stream, &name, &topic_options)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(())
})
}
@@ -807,7 +784,7 @@ impl IggyClient {
let topic = inner
.get_topic(&stream_id, &topic_id)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(topic.map(TopicDetails::from))
})
}
@@ -835,7 +812,7 @@ impl IggyClient {
let topics = inner
.get_topics(&stream_id)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(topics.into_iter().map(Topic::from).collect::<Vec<_>>())
})
}
@@ -889,10 +866,7 @@ impl IggyClient {
// Absent stays absent: a key the caller did not pass is left alone
// server-side rather than reset to a default.
let compression_algorithm = compression_algorithm
- .map(|algo| {
- CompressionAlgorithm::from_str(&algo)
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))
- })
+ .map(|algo| CompressionAlgorithm::from_str(&algo).map_err(to_runtime_error))
.transpose()?;
let update_options = TopicUpdateOptions {
compression_algorithm,
@@ -909,7 +883,7 @@ impl IggyClient {
inner
.update_topic(&stream_id, &topic_id, &name, &update_options)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(())
})
}
@@ -940,7 +914,7 @@ impl IggyClient {
inner
.delete_topic(&stream_id, &topic_id)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(())
})
}
@@ -971,7 +945,93 @@ impl IggyClient {
inner
.purge_topic(&stream_id, &topic_id)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
+ Ok(())
+ })
+ }
+
+ /// Create partitions for a topic. New partition IDs continue from one past the
+ /// current highest ID; IDs removed by deletion can be reused. Existing consumer
+ /// groups are immediately rebalanced across all partitions, advancing their
+ /// generation and dropping pending revocations.
+ ///
+ /// Args:
+ /// stream_id: Stream identifier as `str | int`.
+ /// topic_id: Topic identifier as `str | int`.
+ /// partitions_count: Number of partitions to create as `int`, between 1 and
+ /// 1000 inclusive.
+ ///
+ /// Returns:
+ /// An awaitable that resolves to `None` when the partitions are committed;
+ /// storage materialization completes asynchronously.
+ ///
+ /// Raises:
+ /// ValueError: If an identifier is invalid.
+ /// OverflowError: If `partitions_count` is outside the unsigned 32-bit range.
+ /// RuntimeError: If the client is not authenticated, lacks global
+ /// `manage_streams` or `manage_topics`, per-stream `manage_stream` or
+ /// `manage_topics`, or per-topic `manage_topic` permission, or the
+ /// request fails.
+ #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))]
+ fn create_partitions<'a>(
+ &self,
+ py: Python<'a>,
+ stream_id: PyIdentifier,
+ topic_id: PyIdentifier,
+ partitions_count: u32,
+ ) -> PyResult<Bound<'a, PyAny>> {
+ let stream_id = Identifier::try_from(stream_id)?;
+ let topic_id = Identifier::try_from(topic_id)?;
+ let inner = self.inner.clone();
+
+ future_into_py(py, async move {
+ inner
+ .create_partitions(&stream_id, &topic_id, partitions_count)
+ .await
+ .map_err(to_runtime_error)?;
+ Ok(())
+ })
+ }
+
+ /// Delete the last partitions from a topic, including all messages stored in them.
+ /// Existing consumer groups are immediately rebalanced across the remaining
+ /// partitions, advancing their generation and dropping pending revocations.
+ ///
+ /// Args:
+ /// stream_id: Stream identifier as `str | int`.
+ /// topic_id: Topic identifier as `str | int`.
+ /// partitions_count: Number of partitions to delete as `int` from the end of
+ /// the topic; must be between 1 and 1000 inclusive and no greater than
+ /// its current count.
+ ///
+ /// Returns:
+ /// An awaitable that resolves to `None` when deletion is accepted; storage
+ /// teardown completes asynchronously.
+ ///
+ /// Raises:
+ /// ValueError: If an identifier is invalid.
+ /// OverflowError: If `partitions_count` is outside the unsigned 32-bit range.
+ /// RuntimeError: If the client is not authenticated, lacks global
+ /// `manage_streams` or `manage_topics`, per-stream `manage_stream` or
+ /// `manage_topics`, or per-topic `manage_topic` permission, or the
+ /// request fails.
+ #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))]
+ fn delete_partitions<'a>(
+ &self,
+ py: Python<'a>,
+ stream_id: PyIdentifier,
+ topic_id: PyIdentifier,
+ partitions_count: u32,
+ ) -> PyResult<Bound<'a, PyAny>> {
+ let stream_id = Identifier::try_from(stream_id)?;
+ let topic_id = Identifier::try_from(topic_id)?;
+ let inner = self.inner.clone();
+
+ future_into_py(py, async move {
+ inner
+ .delete_partitions(&stream_id, &topic_id, partitions_count)
+ .await
+ .map_err(to_runtime_error)?;
Ok(())
})
}
@@ -1005,7 +1065,7 @@ impl IggyClient {
inner
.create_consumer_group(&stream_id, &topic_id, &name)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(())
})
}
@@ -1041,7 +1101,7 @@ impl IggyClient {
let group = inner
.get_consumer_group(&stream_id, &topic_id, &group_id)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(group.map(PyConsumerGroupDetails::from))
})
}
@@ -1073,7 +1133,7 @@ impl IggyClient {
let groups = inner
.get_consumer_groups(&stream_id, &topic_id)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(groups
.into_iter()
.map(PyConsumerGroup::from)
@@ -1111,7 +1171,7 @@ impl IggyClient {
inner
.delete_consumer_group(&stream_id, &topic_id, &group_id)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(())
})
}
@@ -1149,7 +1209,7 @@ impl IggyClient {
inner
.join_consumer_group(&stream_id, &topic_id, &group_id)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(())
})
}
@@ -1189,7 +1249,7 @@ impl IggyClient {
inner
.leave_consumer_group(&stream_id, &topic_id, &group_id)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(())
})
}
@@ -1247,7 +1307,7 @@ impl IggyClient {
let response = inner
.send_messages(&stream, &topic, &partitioning, messages.as_mut())
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(PySendMessagesResponse::from(response))
})
}
@@ -1289,7 +1349,7 @@ impl IggyClient {
auto_commit,
)
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
let partition_id = polled_messages.partition_id;
let messages = polled_messages
.messages
@@ -1365,7 +1425,7 @@ impl IggyClient {
let mut builder = self
.inner
.consumer_group(name, stream, topic)
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?
+ .map_err(to_runtime_error)?
.without_encryptor()
.partition(partition_id);
@@ -1425,10 +1485,7 @@ impl IggyClient {
let mut consumer = builder.build();
future_into_py(py, async move {
- consumer
- .init()
- .await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ consumer.init().await.map_err(to_runtime_error)?;
let state = consumer.state();
let name = consumer.name().to_string();
let stream = PyIdentifier::try_from(consumer.stream())?;
@@ -1469,7 +1526,7 @@ impl IggyClient {
let response = inner
.send_binary_request(code, Bytes::from(payload))
.await
- .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
+ .map_err(to_runtime_error)?;
Ok(Python::attach(|py| PyBytes::new(py, &response).unbind()))
})
}
diff --git a/foreign/python/tests/test_partition.py b/foreign/python/tests/test_partition.py
new file mode 100644
index 000000000..193b3cb42
--- /dev/null
+++ b/foreign/python/tests/test_partition.py
@@ -0,0 +1,228 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import asyncio
+
+import pytest
+
+from apache_iggy import (
+ IggyClient,
+ Permissions,
+ SendMessage,
+ StreamPermissions,
+ TopicPermissions,
+)
+
+from .utils import login_fresh_client, unique_credentials
+
+
+async def _create_topic(iggy_client: IggyClient, unique_name):
+ stream_name = unique_name()
+ topic_name = unique_name()
+
+ await iggy_client.create_stream(stream_name)
+ await iggy_client.create_topic(
+ stream=stream_name, name=topic_name, partitions_count=2
+ )
+ return stream_name, topic_name
+
+
+async def _wait_for_messages(iggy_client, stream_id, topic_id, expected):
+ for _ in range(100):
+ topic = await iggy_client.get_topic(stream_id, topic_id)
+ if topic is not None and topic.messages_count == expected:
+ return topic
+ await asyncio.sleep(0.01)
+ raise AssertionError(f"topic messages_count did not reach {expected}")
+
+
+class TestPartitionManagement:
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("numeric_ids", [False, True])
+ async def test_create_and_delete_partitions(
+ self, iggy_client: IggyClient, unique_name, numeric_ids: bool
+ ):
+ stream_name, topic_name = await _create_topic(iggy_client, unique_name)
+ stream = await iggy_client.get_stream(stream_name)
+ assert stream is not None
+ topic = await iggy_client.get_topic(stream.id, topic_name)
+ assert topic is not None
+ stream_id = stream.id if numeric_ids else stream_name
+ topic_id = topic.id if numeric_ids else topic_name
+
+ await iggy_client.create_partitions(stream_id, topic_id, 2)
+ created = await iggy_client.get_topic(stream_id, topic_id)
+ assert created is not None
+ assert created.partitions_count == 4
+ assert [partition.id for partition in created.partitions] == [0, 1, 2, 3]
+
+ await iggy_client.delete_partitions(stream_id, topic_id, 2)
+ deleted = await iggy_client.get_topic(stream_id, topic_id)
+ assert deleted is not None
+ assert deleted.partitions_count == 2
+ assert [partition.id for partition in deleted.partitions] == [0, 1]
+
+ @pytest.mark.asyncio
+ async def test_delete_partitions_rolls_back_stats(
+ self, iggy_client: IggyClient, unique_name
+ ):
+ stream_name, topic_name = await _create_topic(iggy_client, unique_name)
+ await iggy_client.create_partitions(stream_name, topic_name, 2)
+ await iggy_client.send_messages(
+ stream_name, topic_name, 0, [SendMessage("retained")]
+ )
+ await iggy_client.send_messages(
+ stream_name, topic_name, 3, [SendMessage("deleted")]
+ )
+ await _wait_for_messages(iggy_client, stream_name, topic_name, 2)
+
+ await iggy_client.delete_partitions(stream_name, topic_name, 2)
+ deleted = await _wait_for_messages(iggy_client, stream_name, topic_name, 1)
+ assert [partition.id for partition in deleted.partitions] == [0, 1]
+ assert deleted.partitions[0].messages_count == 1
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("method", ["create_partitions", "delete_partitions"])
+ @pytest.mark.parametrize("partitions_count", [0, 1001])
+ async def test_partition_management_rejects_invalid_count(
+ self,
+ iggy_client: IggyClient,
+ unique_name,
+ method: str,
+ partitions_count: int,
+ ):
+ stream_name, topic_name = await _create_topic(iggy_client, unique_name)
+
+ # Zero shares the legacy TooManyPartitions code with an over-limit count.
+ with pytest.raises(RuntimeError, match="Too many partitions"):
+ await getattr(iggy_client, method)(
+ stream_name, topic_name, partitions_count
+ )
+
+ @pytest.mark.asyncio
+ async def test_delete_partitions_rejects_count_larger_than_topic(
+ self, iggy_client: IggyClient, unique_name
+ ):
+ stream_name, topic_name = await _create_topic(iggy_client, unique_name)
+
+ with pytest.raises(RuntimeError, match="Invalid partitions count"):
+ await iggy_client.delete_partitions(stream_name, topic_name, 3)
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("method", ["create_partitions", "delete_partitions"])
+ @pytest.mark.parametrize("missing", ["stream", "topic"])
+ async def test_partition_management_rejects_missing_stream_or_topic(
+ self, iggy_client: IggyClient, unique_name, method: str, missing: str
+ ):
+ stream_name, topic_name = await _create_topic(iggy_client, unique_name)
+ missing_name = unique_name()
+ stream_id = missing_name if missing == "stream" else stream_name
+ topic_id = missing_name if missing == "topic" else topic_name
+
+ with pytest.raises(RuntimeError, match=r"was not found\."):
+ await getattr(iggy_client, method)(stream_id, topic_id, 1)
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("method", ["create_partitions", "delete_partitions"])
+ async def test_partition_management_rejects_invalid_identifier(
+ self, iggy_client: IggyClient, unique_name, method: str
+ ):
+ _, topic_name = await _create_topic(iggy_client, unique_name)
+
+ with pytest.raises(ValueError):
+ await getattr(iggy_client, method)("", topic_name, 1)
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("method", ["create_partitions", "delete_partitions"])
+ @pytest.mark.parametrize("partitions_count", [-1, 2**32])
+ async def test_partition_management_rejects_out_of_range_python_integer(
+ self,
+ iggy_client: IggyClient,
+ unique_name,
+ method: str,
+ partitions_count: int,
+ ):
+ stream_name, topic_name = await _create_topic(iggy_client, unique_name)
+
+ with pytest.raises(OverflowError):
+ await getattr(iggy_client, method)(
+ stream_name, topic_name, partitions_count
+ )
+
+ @pytest.mark.asyncio
+ async def test_delete_partitions_accepts_deleting_all_partitions(
+ self, iggy_client: IggyClient, unique_name
+ ):
+ stream_name, topic_name = await _create_topic(iggy_client, unique_name)
+
+ await iggy_client.delete_partitions(stream_name, topic_name, 2)
+ topic = await iggy_client.get_topic(stream_name, topic_name)
+ assert topic is not None
+ assert topic.partitions_count == 0
+ assert topic.partitions == []
+
+ @pytest.mark.asyncio
+ async def test_partition_management_requires_scoped_manage_topic(
+ self, iggy_client: IggyClient, unique_name
+ ):
+ stream_name, topic_name = await _create_topic(iggy_client, unique_name)
+ other_topic_name = unique_name()
+ await iggy_client.create_topic(
+ stream_name, other_topic_name, partitions_count=2
+ )
+ stream = await iggy_client.get_stream(stream_name)
+ assert stream is not None
+ topic = await iggy_client.get_topic(stream.id, topic_name)
+ other_topic = await iggy_client.get_topic(stream.id, other_topic_name)
+ assert topic is not None
+ assert other_topic is not None
+
+ denied_username, denied_password = unique_credentials(unique_name)
+ denied_user = await iggy_client.create_user(denied_username, denied_password)
+ denied = await login_fresh_client(denied_username, denied_password)
+ for method in ("create_partitions", "delete_partitions"):
+ with pytest.raises(RuntimeError, match="Unauthorized"):
+ await getattr(denied, method)(stream.id, topic.id, 1)
+ unchanged = await iggy_client.get_topic(stream.id, topic.id)
+ assert unchanged is not None
+ assert unchanged.partitions_count == 2
+
+ allowed_username, allowed_password = unique_credentials(unique_name)
+ allowed_user = await iggy_client.create_user(
+ allowed_username,
+ allowed_password,
+ permissions=Permissions(
+ streams={
+ stream.id: StreamPermissions(
+ topics={topic.id: TopicPermissions(manage_topic=True)}
+ )
+ }
+ ),
+ )
+ allowed = await login_fresh_client(allowed_username, allowed_password)
+ await allowed.create_partitions(stream.id, topic.id, 1)
+ await allowed.delete_partitions(stream.id, topic.id, 1)
+ for method in ("create_partitions", "delete_partitions"):
+ with pytest.raises(RuntimeError, match="Unauthorized"):
+ await getattr(allowed, method)(stream.id, other_topic.id, 1)
+ scoped = await iggy_client.get_topic(stream.id, topic.id)
+ untouched = await iggy_client.get_topic(stream.id, other_topic.id)
+ assert scoped is not None and scoped.partitions_count == 2
+ assert untouched is not None and untouched.partitions_count == 2
+
+ await iggy_client.delete_user(denied_user.id)
+ await iggy_client.delete_user(allowed_user.id)