You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tsfile.apache.org by ha...@apache.org on 2024/10/18 02:36:38 UTC
(tsfile) branch iotdb updated (a1d3ab46 -> c82ab892)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch iotdb
in repository https://gitbox.apache.org/repos/asf/tsfile.git
from a1d3ab46 change version to d28fef17
new 4bf9c3e3 Support building python whl package by maven (#265)
new 1eb2078f Fix ColumnFactory missing new data types (#267)
new c82ab892 change version to 1eb2078
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/unit-test.yml | 8 ++++
.gitignore | 6 +++
cpp/pom.xml | 2 +-
java/common/pom.xml | 2 +-
java/examples/pom.xml | 4 +-
java/pom.xml | 4 +-
java/tools/pom.xml | 6 +--
java/tsfile/pom.xml | 4 +-
.../read/common/block/column/ColumnFactory.java | 4 ++
pom.xml | 21 +++-------
python/pom.xml | 26 +++++++++++-
python/requirements.txt | 1 +
python/setup.py | 49 ++++++++++++----------
13 files changed, 87 insertions(+), 50 deletions(-)
(tsfile) 01/03: Support building python whl package by maven (#265)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch iotdb
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 4bf9c3e3661ee1755e308684f6f9489ec4655b07
Author: Haonan <hh...@outlook.com>
AuthorDate: Fri Oct 18 09:58:01 2024 +0800
Support building python whl package by maven (#265)
* fix python package
* package whl by maven
* finish build
* update version
* upload whl
* try package
* try package
* try package
* try fix win
* try fix win
* try fix win
* try fix win
* finish
* fix verify
* fix pkg missing libtsfile
* fix pkg missing libtsfile
---
.github/workflows/unit-test.yml | 8 +++++++
.gitignore | 6 +++++
pom.xml | 19 +++++-----------
python/pom.xml | 24 +++++++++++++++++++-
python/requirements.txt | 1 +
python/setup.py | 49 +++++++++++++++++++++++------------------
6 files changed, 70 insertions(+), 37 deletions(-)
diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml
index ec0e7782..c3c10a0c 100644
--- a/.github/workflows/unit-test.yml
+++ b/.github/workflows/unit-test.yml
@@ -84,3 +84,11 @@ jobs:
shell: bash
run: |
./mvnw${{ steps.platform_suffix.outputs.platform_suffix }} -P with-java,with-cpp,with-python clean verify
+
+ - name: Upload whl Artifact
+ if: ${{ matrix.java == '17'}}
+ uses: actions/upload-artifact@v4
+ with:
+ name: tsfile-${{ runner.os }}-whl
+ path: python/dist/tsfile-*.whl
+ retention-days: 1
diff --git a/.gitignore b/.gitignore
index c2c059c0..2600d5a7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,6 +21,9 @@ docs/pnpm-lock.yaml
# python files
python/build
+python/dist
+python/__pycache__
+python/tsfile.egg-info
python/tsfile/__pycache__
python/tsfile/*so*
python/tsfile/*dll*
@@ -32,3 +35,6 @@ python/venv/*
cpp/cmake-build-debug-mingw/
.vscode/settings.json
+.vscode/
+
+build/*
diff --git a/pom.xml b/pom.xml
index 314a995d..84523135 100644
--- a/pom.xml
+++ b/pom.xml
@@ -125,6 +125,10 @@
<!-- generated by cython-->
<exclude>**/tsfile/tsfile_pywrapper.cpp</exclude>
<exclude>**/venv/**</exclude>
+ <!-- generated by wheel-->
+ <exclude>**/tsfile.egg-info/**</exclude>
+ <!-- Exclude third_party-->
+ <exclude>**/third_party/**</exclude>
</excludes>
</configuration>
</plugin>
@@ -597,6 +601,7 @@
<profile>
<id>with-python</id>
<modules>
+ <module>cpp</module>
<module>python</module>
</modules>
</profile>
@@ -658,7 +663,6 @@
<cmake.generator>Unix Makefiles</cmake.generator>
<python.venv.bin>venv/bin/</python.venv.bin>
<python.exe.bin>python3</python.exe.bin>
- <cython.compiler>unix</cython.compiler>
</properties>
</profile>
<!-- Profile for linux amd64 (mainly AMD Processors) (Self-Enabling) -->
@@ -677,7 +681,6 @@
<cmake.generator>Unix Makefiles</cmake.generator>
<python.venv.bin>venv/bin/</python.venv.bin>
<python.exe.bin>python3</python.exe.bin>
- <cython.compiler>unix</cython.compiler>
</properties>
</profile>
<!-- Profile for linux aarch64 (mainly ARM64 Processors) (Self-Enabling) -->
@@ -696,7 +699,6 @@
<cmake.generator>Unix Makefiles</cmake.generator>
<python.venv.bin>venv/bin/</python.venv.bin>
<python.exe.bin>python3</python.exe.bin>
- <cython.compiler>unix</cython.compiler>
</properties>
</profile>
<!-- Profile for mac x86_64 (mainly Intel Processors) (Self-Enabling) -->
@@ -714,7 +716,6 @@
<cmake.generator>Unix Makefiles</cmake.generator>
<python.venv.bin>venv/bin/</python.venv.bin>
<python.exe.bin>python3</python.exe.bin>
- <cython.compiler>unix</cython.compiler>
</properties>
</profile>
<!-- Profile for mac aarch64 (mainly new Mac Processors) (Self-Enabling) -->
@@ -732,7 +733,6 @@
<cmake.generator>Unix Makefiles</cmake.generator>
<python.venv.bin>venv/bin/</python.venv.bin>
<python.exe.bin>python3</python.exe.bin>
- <cython.compiler>unix</cython.compiler>
</properties>
</profile>
<!-- profile for windows x86_64 (mainly Intel Processors) (Self-Enabling) -->
@@ -748,11 +748,8 @@
<os.suffix>win</os.suffix>
<os.classifier>windows-x86_64</os.classifier>
<cmake.generator>MinGW Makefiles</cmake.generator>
- <!-- The generated code relied on Boost and that relies on VS and can't be built with MinGW -->
- <!--cmake.generator>Visual Studio 17 2022</cmake.generator-->
<python.venv.bin>venv/Scripts/</python.venv.bin>
<python.exe.bin>python</python.exe.bin>
- <cython.compiler>mingw32</cython.compiler>
</properties>
</profile>
<!-- profile for windows amd64 (mainly AMD Processors) (Self-Enabling) -->
@@ -768,11 +765,8 @@
<os.suffix>win</os.suffix>
<os.classifier>windows-amd64</os.classifier>
<cmake.generator>MinGW Makefiles</cmake.generator>
- <!-- The generated code relied on Boost and that relies on VS and can't be built with MinGW -->
- <!--cmake.generator>Visual Studio 17 2022</cmake.generator-->
<python.venv.bin>venv/Scripts/</python.venv.bin>
<python.exe.bin>python</python.exe.bin>
- <cython.compiler>mingw32</cython.compiler>
</properties>
</profile>
<!-- profile for windows aarch64 (mainly VM on newer Mac) (Self-Enabling) -->
@@ -788,11 +782,8 @@
<os.suffix>win</os.suffix>
<os.classifier>windows-aarch64</os.classifier>
<cmake.generator>MinGW Makefiles</cmake.generator>
- <!-- The generated code relied on Boost and that relies on VS and can't be built with MinGW -->
- <!--cmake.generator>Visual Studio 17 2022</cmake.generator-->
<python.venv.bin>venv/Scripts/</python.venv.bin>
<python.exe.bin>python</python.exe.bin>
- <cython.compiler>mingw32</cython.compiler>
</properties>
</profile>
<!-- Little helper profile that will disable running the cmake tests when the maven tests are being skipped -->
diff --git a/python/pom.xml b/python/pom.xml
index 0d0d9999..d80f1b81 100644
--- a/python/pom.xml
+++ b/python/pom.xml
@@ -103,7 +103,6 @@
<argument>setup.py</argument>
<argument>build_ext</argument>
<argument>--inplace</argument>
- <argument>--compiler=${cython.compiler}</argument>
</arguments>
</configuration>
</execution>
@@ -136,6 +135,20 @@
</arguments>
</configuration>
</execution>
+ <execution>
+ <id>build-whl</id>
+ <phase>package</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <executable>${python.venv.bin}${python.exe.bin}</executable>
+ <arguments>
+ <argument>setup.py</argument>
+ <argument>bdist_wheel</argument>
+ </arguments>
+ </configuration>
+ </execution>
</executions>
</plugin>
<plugin>
@@ -152,6 +165,15 @@
<include>*.cpp</include>
</includes>
</fileset>
+ <fileset>
+ <directory>tsfile.egg-info</directory>
+ </fileset>
+ <fileset>
+ <directory>build</directory>
+ </fileset>
+ <fileset>
+ <directory>dist</directory>
+ </fileset>
</filesets>
</configuration>
</plugin>
diff --git a/python/requirements.txt b/python/requirements.txt
index 0c82a92b..1f579636 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -21,4 +21,5 @@ cython==3.0.10
numpy==1.26.4
pandas==2.2.2
setuptools==70.0.0
+wheel
diff --git a/python/setup.py b/python/setup.py
index bd438823..01120060 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -24,28 +24,32 @@ import platform
import shutil
import os
+system = platform.system()
-def copy_lib_files(system, source_dir, target_dir, suffix):
+def copy_lib_files(source_dir, target_dir, suffix):
lib_file_name = f"libtsfile.{suffix}"
source = os.path.join(source_dir, lib_file_name)
target = os.path.join(target_dir, lib_file_name)
- shutil.copyfile(source, target)
+
+ if os.path.exists(source):
+ shutil.copyfile(source, target)
if system == "Linux":
- link_name = os.path.join(target_dir, f"libtsfile.so")
+ link_name = os.path.join(target_dir, "libtsfile.so")
if os.path.exists(link_name):
os.remove(link_name)
os.symlink(lib_file_name, link_name)
- if system == "Darwin":
- link_name = os.path.join(target_dir, f"libtsfile.dylib")
+ elif system == "Darwin":
+ link_name = os.path.join(target_dir, "libtsfile.dylib")
if os.path.exists(link_name):
os.remove(link_name)
os.symlink(lib_file_name, link_name)
def copy_header(source, target):
- shutil.copyfile(source, target)
+ if os.path.exists(source):
+ shutil.copyfile(source, target)
class BuildExt(build_ext):
@@ -55,20 +59,18 @@ class BuildExt(build_ext):
ext.include_dirs.append(numpy_include)
super().build_extensions()
+ def finalize_options(self):
+ if platform.system() == "Windows":
+ self.compiler = 'mingw32'
+ super().finalize_options()
+
+
+project_dir = os.path.dirname(os.path.abspath(__file__))
-project_dir = os.path.dirname(__file__)
libtsfile_shard_dir = os.path.join(project_dir, "..", "cpp", "target", "build", "lib")
libtsfile_dir = os.path.join(project_dir, "tsfile")
include_dir = os.path.join(project_dir, "tsfile")
-source_file = os.path.join(project_dir, "tsfile", "tsfile_pywrapper.pyx")
-
-if platform.system() == "Darwin":
- copy_lib_files("Darwin", libtsfile_shard_dir, libtsfile_dir, "1.0.dylib")
-elif platform.system() == "Linux":
- copy_lib_files("Linux", libtsfile_shard_dir, libtsfile_dir, "so.1.0")
-else:
- copy_lib_files("Windows", libtsfile_shard_dir, libtsfile_dir, "dll")
-
+source_file = os.path.join("tsfile", "tsfile_pywrapper.pyx")
source_include_dir = os.path.join(
project_dir, "..", "cpp", "src", "cwrapper", "TsFile-cwrapper.h"
@@ -76,6 +78,13 @@ source_include_dir = os.path.join(
target_include_dir = os.path.join(project_dir, "tsfile", "TsFile-cwrapper.h")
copy_header(source_include_dir, target_include_dir)
+if system == "Darwin":
+ copy_lib_files(libtsfile_shard_dir, libtsfile_dir, "1.0.dylib")
+elif system == "Linux":
+ copy_lib_files(libtsfile_shard_dir, libtsfile_dir, "so.1.0")
+else:
+ copy_lib_files(libtsfile_shard_dir, libtsfile_dir, "dll")
+
ext_modules_tsfile = [
Extension(
"tsfile.tsfile_pywrapper",
@@ -91,7 +100,7 @@ ext_modules_tsfile = [
setup(
name="tsfile",
- version="0.1",
+ version="1.2.0.dev0",
description="Tsfile reader and writer for python",
url="https://tsfile.apache.org",
author='"Apache TsFile"',
@@ -102,11 +111,7 @@ setup(
include_dirs=[np.get_include()],
package_data={
"tsfile": [
- os.path.join("*tsfile", "*.so*"),
- os.path.join("*tsfile", "*.dylib"),
- os.path.join("*tsfile", "*.pyd"),
- os.path.join("*tsfile", "*.dll"),
- os.path.join("tsfile", "tsfile.py"),
+ "libtsfile.*",
]
},
include_package_data=True,
(tsfile) 02/03: Fix ColumnFactory missing new data types (#267)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch iotdb
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 1eb2078f0ac725d1b405e51e85800514c8b7aaa3
Author: Haonan <hh...@outlook.com>
AuthorDate: Fri Oct 18 10:06:26 2024 +0800
Fix ColumnFactory missing new data types (#267)
---
.../org/apache/tsfile/read/common/block/column/ColumnFactory.java | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java
index 42984c3d..88a40113 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/ColumnFactory.java
@@ -26,14 +26,18 @@ public class ColumnFactory {
public static Column create(TSDataType dataType, int initialCapacity) {
switch (dataType) {
case INT64:
+ case TIMESTAMP:
return new LongColumn(initialCapacity);
case DOUBLE:
return new DoubleColumn(initialCapacity);
case FLOAT:
return new FloatColumn(initialCapacity);
case TEXT:
+ case STRING:
+ case BLOB:
return new BinaryColumn(initialCapacity);
case INT32:
+ case DATE:
return new IntColumn(initialCapacity);
case BOOLEAN:
return new BooleanColumn(initialCapacity);
(tsfile) 03/03: change version to 1eb2078
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch iotdb
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit c82ab89203584df82bfa9e727073ac4ebdede5d4
Author: HTHou <hh...@outlook.com>
AuthorDate: Fri Oct 18 10:36:21 2024 +0800
change version to 1eb2078
---
cpp/pom.xml | 2 +-
java/common/pom.xml | 2 +-
java/examples/pom.xml | 4 ++--
java/pom.xml | 4 ++--
java/tools/pom.xml | 6 +++---
java/tsfile/pom.xml | 4 ++--
pom.xml | 2 +-
python/pom.xml | 2 +-
8 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/cpp/pom.xml b/cpp/pom.xml
index 2b090972..ba33d8d4 100644
--- a/cpp/pom.xml
+++ b/cpp/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile-parent</artifactId>
- <version>1.2.0-d28fef17-SNAPSHOT</version>
+ <version>1.2.0-1eb2078-SNAPSHOT</version>
</parent>
<artifactId>tsfile-cpp</artifactId>
<packaging>pom</packaging>
diff --git a/java/common/pom.xml b/java/common/pom.xml
index c241664b..a4fd7204 100644
--- a/java/common/pom.xml
+++ b/java/common/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile-java</artifactId>
- <version>1.2.0-d28fef17-SNAPSHOT</version>
+ <version>1.2.0-1eb2078-SNAPSHOT</version>
</parent>
<artifactId>common</artifactId>
<name>TsFile: Java: Common</name>
diff --git a/java/examples/pom.xml b/java/examples/pom.xml
index 00576802..39d51843 100644
--- a/java/examples/pom.xml
+++ b/java/examples/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile-java</artifactId>
- <version>1.2.0-d28fef17-SNAPSHOT</version>
+ <version>1.2.0-1eb2078-SNAPSHOT</version>
</parent>
<artifactId>examples</artifactId>
<packaging>pom</packaging>
@@ -37,7 +37,7 @@
<dependency>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile</artifactId>
- <version>1.2.0-d28fef17-SNAPSHOT</version>
+ <version>1.2.0-1eb2078-SNAPSHOT</version>
</dependency>
</dependencies>
<build>
diff --git a/java/pom.xml b/java/pom.xml
index f31713f7..eb8cd9b2 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -24,10 +24,10 @@
<parent>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile-parent</artifactId>
- <version>1.2.0-d28fef17-SNAPSHOT</version>
+ <version>1.2.0-1eb2078-SNAPSHOT</version>
</parent>
<artifactId>tsfile-java</artifactId>
- <version>1.2.0-d28fef17-SNAPSHOT</version>
+ <version>1.2.0-1eb2078-SNAPSHOT</version>
<packaging>pom</packaging>
<name>TsFile: Java</name>
<modules>
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index 30da1e49..77654590 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile-java</artifactId>
- <version>1.2.0-d28fef17-SNAPSHOT</version>
+ <version>1.2.0-1eb2078-SNAPSHOT</version>
</parent>
<artifactId>tools</artifactId>
<name>TsFile: Java: Tools</name>
@@ -32,7 +32,7 @@
<dependency>
<groupId>org.apache.tsfile</groupId>
<artifactId>common</artifactId>
- <version>1.2.0-d28fef17-SNAPSHOT</version>
+ <version>1.2.0-1eb2078-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
@@ -50,7 +50,7 @@
<dependency>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile</artifactId>
- <version>1.2.0-d28fef17-SNAPSHOT</version>
+ <version>1.2.0-1eb2078-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
diff --git a/java/tsfile/pom.xml b/java/tsfile/pom.xml
index 757bafcf..408b6d54 100644
--- a/java/tsfile/pom.xml
+++ b/java/tsfile/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile-java</artifactId>
- <version>1.2.0-d28fef17-SNAPSHOT</version>
+ <version>1.2.0-1eb2078-SNAPSHOT</version>
</parent>
<artifactId>tsfile</artifactId>
<name>TsFile: Java: TsFile</name>
@@ -38,7 +38,7 @@
<dependency>
<groupId>org.apache.tsfile</groupId>
<artifactId>common</artifactId>
- <version>1.2.0-d28fef17-SNAPSHOT</version>
+ <version>1.2.0-1eb2078-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.github.luben</groupId>
diff --git a/pom.xml b/pom.xml
index 84523135..916db732 100644
--- a/pom.xml
+++ b/pom.xml
@@ -28,7 +28,7 @@
</parent>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile-parent</artifactId>
- <version>1.2.0-d28fef17-SNAPSHOT</version>
+ <version>1.2.0-1eb2078-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache TsFile Project Parent POM</name>
<properties>
diff --git a/python/pom.xml b/python/pom.xml
index d80f1b81..7d21995d 100644
--- a/python/pom.xml
+++ b/python/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.tsfile</groupId>
<artifactId>tsfile-parent</artifactId>
- <version>1.2.0-d28fef17-SNAPSHOT</version>
+ <version>1.2.0-1eb2078-SNAPSHOT</version>
</parent>
<artifactId>tsfile-python</artifactId>
<packaging>pom</packaging>