You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2022/04/07 06:43:36 UTC
[iotdb] 01/02: [IOTDB-2859] Fix python tablet with None value is incorrect
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch fix_py_nullable_tablet
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f91cc8b7ace252fa4e0b6553ad266b6fd372dfb5
Author: HTHou <hh...@outlook.com>
AuthorDate: Thu Apr 7 14:42:09 2022 +0800
[IOTDB-2859] Fix python tablet with None value is incorrect
---
client-py/iotdb/utils/Tablet.py | 27 ++++++------
client-py/tests/test_tablet.py | 98 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 112 insertions(+), 13 deletions(-)
diff --git a/client-py/iotdb/utils/Tablet.py b/client-py/iotdb/utils/Tablet.py
index 2f41889f22..606164e6a2 100644
--- a/client-py/iotdb/utils/Tablet.py
+++ b/client-py/iotdb/utils/Tablet.py
@@ -94,7 +94,7 @@ class Tablet(object):
has_none = False
for i in range(self.__column_number):
bitmap = None
- bitmaps.insert(i, bitmap)
+ bitmaps.append(bitmap)
if self.__data_types[i] == TSDataType.BOOLEAN:
format_str_list.append(str(self.__row_number))
format_str_list.append("?")
@@ -103,7 +103,7 @@ class Tablet(object):
values_tobe_packed.append(self.__values[j][i])
else:
values_tobe_packed.append(False)
- self.__mark_none_value(bitmaps, bitmap, i, j)
+ self.__mark_none_value(bitmaps, i, j)
has_none = True
elif self.__data_types[i] == TSDataType.INT32:
@@ -114,7 +114,7 @@ class Tablet(object):
values_tobe_packed.append(self.__values[j][i])
else:
values_tobe_packed.append(0)
- self.__mark_none_value(bitmaps, bitmap, i, j)
+ self.__mark_none_value(bitmaps, i, j)
has_none = True
elif self.__data_types[i] == TSDataType.INT64:
@@ -125,7 +125,7 @@ class Tablet(object):
values_tobe_packed.append(self.__values[j][i])
else:
values_tobe_packed.append(0)
- self.__mark_none_value(bitmaps, bitmap, i, j)
+ self.__mark_none_value(bitmaps, i, j)
has_none = True
elif self.__data_types[i] == TSDataType.FLOAT:
@@ -136,7 +136,7 @@ class Tablet(object):
values_tobe_packed.append(self.__values[j][i])
else:
values_tobe_packed.append(0)
- self.__mark_none_value(bitmaps, bitmap, i, j)
+ self.__mark_none_value(bitmaps, i, j)
has_none = True
elif self.__data_types[i] == TSDataType.DOUBLE:
@@ -147,7 +147,7 @@ class Tablet(object):
values_tobe_packed.append(self.__values[j][i])
else:
values_tobe_packed.append(0)
- self.__mark_none_value(bitmaps, bitmap, i, j)
+ self.__mark_none_value(bitmaps, i, j)
has_none = True
elif self.__data_types[i] == TSDataType.TEXT:
@@ -166,11 +166,13 @@ class Tablet(object):
format_str_list.append("s")
values_tobe_packed.append(len(value_bytes))
values_tobe_packed.append(value_bytes)
- self.__mark_none_value(bitmaps, bitmap, i, j)
+ self.__mark_none_value(bitmaps, i, j)
has_none = True
else:
- raise RuntimeError("Unsupported data type:" + str(self.__data_types[i]))
+ raise RuntimeError(
+ "Unsupported data type:" + str(self.__data_types[i])
+ )
if has_none:
for i in range(self.__column_number):
@@ -186,8 +188,7 @@ class Tablet(object):
format_str = "".join(format_str_list)
return struct.pack(format_str, *values_tobe_packed)
- def __mark_none_value(self, bitmaps, bitmap, column, row):
- if bitmap is None:
- bitmap = BitMap(self.__row_number)
- bitmaps.insert(column, bitmap)
- bitmap.mark(row)
+ def __mark_none_value(self, bitmaps, column, row):
+ if bitmaps[column] is None:
+ bitmaps[column] = BitMap(self.__row_number)
+ bitmaps[column].mark(row)
diff --git a/client-py/tests/test_tablet.py b/client-py/tests/test_tablet.py
new file mode 100644
index 0000000000..2d4da04c86
--- /dev/null
+++ b/client-py/tests/test_tablet.py
@@ -0,0 +1,98 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import pandas as pd
+from pandas.testing import assert_frame_equal
+
+from iotdb.IoTDBContainer import IoTDBContainer
+from iotdb.Session import Session
+from iotdb.utils.IoTDBConstants import TSDataType
+from iotdb.utils.Tablet import Tablet
+
+
+def test_tablet_insertion():
+ with IoTDBContainer("iotdb:dev") as db:
+ db: IoTDBContainer
+ session = Session(db.get_container_host_ip(), db.get_exposed_port(6667))
+ session.open(False)
+
+ measurements_ = ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"]
+ data_types_ = [
+ TSDataType.BOOLEAN,
+ TSDataType.INT32,
+ TSDataType.INT64,
+ TSDataType.FLOAT,
+ TSDataType.DOUBLE,
+ TSDataType.TEXT,
+ ]
+ values_ = [
+ [False, 10, 11, 1.1, 10011.1, "test01"],
+ [True, 100, 11111, 1.25, 101.0, "test02"],
+ [False, 100, 1, 188.1, 688.25, "test03"],
+ [True, 0, 0, 0, 6.25, "test04"],
+ ]
+ timestamps_ = [16, 17, 18, 19]
+ tablet_ = Tablet(
+ "root.sg_test_01.d_01", measurements_, data_types_, values_, timestamps_
+ )
+ df_input = pd.DataFrame(values_)
+ session.insert_tablet(tablet_)
+ df_input.insert(0, "Time", timestamps_)
+
+ session_data_set = session.execute_query_statement("select s_01, s_02, s_03, s_04, s_05, s_06 from root.sg_test_01.d_01")
+ df_output = session_data_set.todf()
+ df_output = df_output[df_input.columns.tolist()]
+
+ session.close()
+ assert_frame_equal(df_input, df_output)
+
+def test_nullable_tablet_insertion():
+ with IoTDBContainer("iotdb:dev") as db:
+ db: IoTDBContainer
+ session = Session(db.get_container_host_ip(), db.get_exposed_port(6667))
+ session.open(False)
+
+ measurements_ = ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"]
+ data_types_ = [
+ TSDataType.BOOLEAN,
+ TSDataType.INT32,
+ TSDataType.INT64,
+ TSDataType.FLOAT,
+ TSDataType.DOUBLE,
+ TSDataType.TEXT,
+ ]
+ values_ = [
+ [None, None, 11, 1.1, 10011.1, "test01"],
+ [True, None, 11111, 1.25, 101.0, "test02"],
+ [False, 100, 1, None, 688.25, "test03"],
+ [True, None, 0, 0, 6.25, None],
+ ]
+ timestamps_ = [16, 17, 18, 19]
+ tablet_ = Tablet(
+ "root.sg_test_01.d_01", measurements_, data_types_, values_, timestamps_
+ )
+ df_input = pd.DataFrame(values_)
+ session.insert_tablet(tablet_)
+ df_input.insert(0, "Time", timestamps_)
+
+ session_data_set = session.execute_query_statement("select s_01, s_02, s_03, s_04, s_05, s_06 from root.sg_test_01.d_01")
+ df_output = session_data_set.todf()
+ df_output = df_output[df_input.columns.tolist()]
+
+ session.close()
+ assert_frame_equal(df_input, df_output)