You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2025/03/26 00:37:42 UTC
(iotdb) branch query_v4_py created (now 8c34ba0f7dc)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch query_v4_py
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 8c34ba0f7dc query_v4-py
This branch includes the following new commits:
new 345188c4fdf try optimize todf
new 8c34ba0f7dc query_v4-py
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 02/02: query_v4-py
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch query_v4_py
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8c34ba0f7dce48f07b36fea1deca022a5a472d3a
Author: HTHou <hh...@outlook.com>
AuthorDate: Wed Mar 26 08:37:20 2025 +0800
query_v4-py
---
iotdb-client/client-py/iotdb/tsfile/utils/tsblock_serde.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/iotdb-client/client-py/iotdb/tsfile/utils/tsblock_serde.py b/iotdb-client/client-py/iotdb/tsfile/utils/tsblock_serde.py
index abdd19e0d0b..854be120ffb 100644
--- a/iotdb-client/client-py/iotdb/tsfile/utils/tsblock_serde.py
+++ b/iotdb-client/client-py/iotdb/tsfile/utils/tsblock_serde.py
@@ -74,7 +74,7 @@ def read_from_buffer(buffer, size):
def read_column_types(buffer, value_column_count):
data_types = np.frombuffer(buffer, dtype=np.uint8, count=value_column_count)
new_buffer = buffer[value_column_count:]
- if not np.all(np.isin(data_types, [0, 1, 2, 3, 4, 5])):
+ if not np.all(np.isin(data_types, (0, 1, 2, 3, 4, 5))):
raise Exception("Invalid data type encountered: " + str(data_types))
return data_types, new_buffer
(iotdb) 01/02: try optimize todf
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch query_v4_py
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 345188c4fdf017d0ddf0504df70a61ba76b92d1c
Author: HTHou <hh...@outlook.com>
AuthorDate: Tue Mar 25 22:43:26 2025 +0800
try optimize todf
---
.../client-py/iotdb/utils/iotdb_rpc_dataset.py | 24 +++++++++++++---------
1 file changed, 14 insertions(+), 10 deletions(-)
diff --git a/iotdb-client/client-py/iotdb/utils/iotdb_rpc_dataset.py b/iotdb-client/client-py/iotdb/utils/iotdb_rpc_dataset.py
index 8530fef5c20..e575a917fca 100644
--- a/iotdb-client/client-py/iotdb/utils/iotdb_rpc_dataset.py
+++ b/iotdb-client/client-py/iotdb/utils/iotdb_rpc_dataset.py
@@ -242,8 +242,11 @@ class IoTDBRpcDataSet(object):
def result_set_to_pandas(self):
result = {}
+ has_pd_series = []
for i in range(len(self.__column_index_2_tsblock_column_index_list)):
result[i] = []
+ has_pd_series.append(False)
+ total_length = 0
while self._has_next_result_set():
time_array, column_arrays, null_indicators, array_length = deserialize(
memoryview(self.__query_result[self.__query_result_index])
@@ -257,6 +260,7 @@ class IoTDBRpcDataSet(object):
)
result[0].append(time_array)
+ total_length += array_length
for i, location in enumerate(
self.__column_index_2_tsblock_column_index_list
):
@@ -280,7 +284,7 @@ class IoTDBRpcDataSet(object):
data_array = np.array([x.decode("utf-8") for x in column_array])
# DATE
elif data_type == 9:
- data_array = pd.Series(column_array).apply(parse_int_to_date)
+ data_array = np.array([parse_int_to_date(x) for x in column_array])
else:
raise RuntimeError("unsupported data type {}.".format(data_type))
@@ -320,10 +324,13 @@ class IoTDBRpcDataSet(object):
if data_type == 1:
tmp_array = pd.Series(tmp_array).astype("Int32")
+ has_pd_series[i] = True
elif data_type == 2 or data_type == 8:
tmp_array = pd.Series(tmp_array).astype("Int64")
+ has_pd_series[i] = True
elif data_type == 0:
tmp_array = pd.Series(tmp_array).astype("boolean")
+ has_pd_series[i] = True
data_array = tmp_array
@@ -332,16 +339,13 @@ class IoTDBRpcDataSet(object):
for k, v in result.items():
if v is None or len(v) < 1 or v[0] is None:
result[k] = []
- elif v[0].dtype == "Int32":
- v = [x if isinstance(x, pd.Series) else pd.Series(x) for x in v]
- result[k] = pd.concat(v, ignore_index=True).astype("Int32")
- elif v[0].dtype == "Int64":
- v = [x if isinstance(x, pd.Series) else pd.Series(x) for x in v]
- result[k] = pd.concat(v, ignore_index=True).astype("Int64")
- elif v[0].dtype == bool:
- result[k] = pd.Series(np.concatenate(v, axis=0)).astype("boolean")
+ elif not has_pd_series[k]:
+ res = np.empty(total_length, dtype=v[0].dtype)
+ np.concatenate(v, axis=0, out=res)
+ result[k] = res
else:
- result[k] = np.concatenate(v, axis=0)
+ v = [x if isinstance(x, pd.Series) else pd.Series(x) for x in v]
+ result[k] = pd.concat(v, ignore_index=True)
df = pd.DataFrame(result)
df.columns = self.__column_name_list