You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2022/04/08 04:26:15 UTC
[iotdb] 01/03: init
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch numpy_type_check
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d9b43753494688cd07762bf3f76fa6382f0a1016
Author: HTHou <hh...@outlook.com>
AuthorDate: Fri Apr 8 11:05:51 2022 +0800
init
---
client-py/iotdb/utils/NumpyTablet.py | 16 ++++-
client-py/tests/test_numpy_tablet.py | 135 +++++++++++++++++++++++++++++++++++
2 files changed, 148 insertions(+), 3 deletions(-)
diff --git a/client-py/iotdb/utils/NumpyTablet.py b/client-py/iotdb/utils/NumpyTablet.py
index 8dd8457459..21e651d2d6 100644
--- a/client-py/iotdb/utils/NumpyTablet.py
+++ b/client-py/iotdb/utils/NumpyTablet.py
@@ -17,7 +17,7 @@
#
import struct
-
+import numpy as np
from iotdb.utils.IoTDBConstants import TSDataType
from iotdb.utils.BitMap import BitMap
@@ -41,15 +41,25 @@ class NumpyTablet(object):
"""
if len(values) > 0 and len(values[0]) != len(timestamps):
raise RuntimeError(
- "Input error! len(timestamps) does not equal to len(values)!"
+ "Input error! len(timestamps) does not equal to len(values[0])!"
+ )
+ if len(values) != len(data_types):
+ raise RuntimeError(
+ "Input error! len(values) does not equal to len(data_types)!"
)
- if not NumpyTablet.check_sorted(timestamps):
+ if not self.check_sorted(timestamps):
index = timestamps.argsort()
timestamps = timestamps[index]
for i in range(len(values)):
values[i] = values[i][index]
+ if timestamps.dtype != np.dtype(">i8"):
+ timestamps = timestamps.astype(np.dtype(">i8"))
+ for i in range(len(values)):
+
+
+
self.__values = values
self.__timestamps = timestamps
self.__device_id = device_id
diff --git a/client-py/tests/test_numpy_tablet.py b/client-py/tests/test_numpy_tablet.py
new file mode 100644
index 0000000000..ea7502ce96
--- /dev/null
+++ b/client-py/tests/test_numpy_tablet.py
@@ -0,0 +1,135 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import numpy as np
+from iotdb.utils.IoTDBConstants import TSDataType
+from iotdb.utils.NumpyTablet import NumpyTablet
+from iotdb.utils.Tablet import Tablet
+
+
+def test_numpy_tablet_serialization():
+
+ measurements_ = ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"]
+ data_types_ = [
+ TSDataType.BOOLEAN,
+ TSDataType.INT32,
+ TSDataType.INT64,
+ TSDataType.FLOAT,
+ TSDataType.DOUBLE,
+ TSDataType.TEXT,
+ ]
+ values_ = [
+ [False, 10, 11, 1.1, 10011.1, "test01"],
+ [True, 100, 11111, 1.25, 101.0, "test02"],
+ [False, 100, 1, 188.1, 688.25, "test03"],
+ [True, 0, 0, 0, 6.25, "test04"],
+ ]
+ timestamps_ = [16, 17, 18, 19]
+ tablet_ = Tablet(
+ "root.sg_test_01.d_01", measurements_, data_types_, values_, timestamps_
+ )
+ np_values_ = [
+ np.array([False, True, False, True], np.dtype(">?")),
+ np.array([10, 100, 100, 0], np.dtype(">i4")),
+ np.array([11, 11111, 1, 0], np.dtype(">i8")),
+ np.array([1.1, 1.25, 188.1, 0], np.dtype(">f4")),
+ np.array([10011.1, 101.0, 688.25, 6.25], np.dtype(">f8")),
+ np.array(["test01", "test02", "test03", "test04"]),
+ ]
+ np_timestamps_ = np.array([16, 17, 18, 19], np.dtype(">i8"))
+ np_tablet_ = NumpyTablet(
+ "root.sg_test_01.d_01", measurements_, data_types_, np_values_, np_timestamps_
+ )
+ assert tablet_.get_binary_timestamps() == np_tablet_.get_binary_timestamps()
+ assert tablet_.get_binary_values() == np_tablet_.get_binary_values()
+
+
+def test_sort_numpy_tablet():
+
+ measurements_ = ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"]
+ data_types_ = [
+ TSDataType.BOOLEAN,
+ TSDataType.INT32,
+ TSDataType.INT64,
+ TSDataType.FLOAT,
+ TSDataType.DOUBLE,
+ TSDataType.TEXT,
+ ]
+ values_ = [
+ [True, 10000, 11111, 8.999, 776, "test05"],
+ [True, 1000, 1111, 0, 6.25, "test06"],
+ [False, 100, 111, 188.1, 688.25, "test07"],
+ [False, 10, 11, 1.25, 101.0, "test08"],
+ [False, 0, 1, 1.1, 10011.1, "test09"],
+ ]
+ timestamps_ = [5, 6, 7, 8, 9]
+ tablet_ = Tablet(
+ "root.sg_test_01.d_01", measurements_, data_types_, values_, timestamps_
+ )
+ np_values_unsorted = [
+ np.array([False, False, False, True, True], np.dtype('>?')),
+ np.array([0, 10, 100, 1000, 10000], np.dtype(">i4")),
+ np.array([1, 11, 111, 1111, 11111], np.dtype(">i8")),
+ np.array([1.1, 1.25, 188.1, 0, 8.999], np.dtype(">f4")),
+ np.array([10011.1, 101.0, 688.25, 6.25, 776], np.dtype(">f8")),
+ np.array(["test09", "test08", "test07", "test06", "test05"]),
+ ]
+ np_timestamps_unsorted = np.array([9, 8, 7, 6, 5], np.dtype(">i8"))
+ np_tablet_ = NumpyTablet(
+ "root.sg_test_01.d_01", measurements_, data_types_, np_values_unsorted, np_timestamps_unsorted
+ )
+ assert tablet_.get_binary_timestamps() == np_tablet_.get_binary_timestamps()
+ assert tablet_.get_binary_values() == np_tablet_.get_binary_values()
+
+def test_numpy_tablet_correct_endian():
+
+ measurements_ = ["s_01", "s_02", "s_03", "s_04", "s_05", "s_06"]
+ data_types_ = [
+ TSDataType.BOOLEAN,
+ TSDataType.INT32,
+ TSDataType.INT64,
+ TSDataType.FLOAT,
+ TSDataType.DOUBLE,
+ TSDataType.TEXT,
+ ]
+ values_ = [
+ [True, 10000, 11111, 8.999, 776, "test05"],
+ [True, 1000, 1111, 0, 6.25, "test06"],
+ [False, 100, 111, 188.1, 688.25, "test07"],
+ [False, 10, 11, 1.25, 101.0, "test08"],
+ [False, 0, 1, 1.1, 10011.1, "test09"],
+ ]
+ timestamps_ = [5, 6, 7, 8, 9]
+ tablet_ = Tablet(
+ "root.sg_test_01.d_01", measurements_, data_types_, values_, timestamps_
+ )
+ np_values_unsorted = [
+ np.array([False, False, False, True, True], np.dtype('>?')),
+ np.array([0, 10, 100, 1000, 10000], np.dtype(">i4")),
+ np.array([1, 11, 111, 1111, 11111], np.dtype(">i8")),
+ np.array([1.1, 1.25, 188.1, 0, 8.999], np.dtype(">f4")),
+ np.array([10011.1, 101.0, 688.25, 6.25, 776], np.dtype(">f8")),
+ np.array(["test09", "test08", "test07", "test06", "test05"]),
+ ]
+ np_timestamps_unsorted = np.array([9, 8, 7, 6, 5])
+ assert np_timestamps_unsorted.dtype != np.dtype(">i8")
+ np_tablet_ = NumpyTablet(
+ "root.sg_test_01.d_01", measurements_, data_types_, np_values_unsorted, np_timestamps_unsorted
+ )
+ assert tablet_.get_binary_timestamps() == np_tablet_.get_binary_timestamps()
+ assert tablet_.get_binary_values() == np_tablet_.get_binary_values()
\ No newline at end of file