[Python] Fix Linux Py3.14 wheel CI and dataframe edge cases (#789)

* fix py3.9 black

* ci wheel python 3.14

* ci fix

* ci fix

* fix ci

* ci fix

* ci fix

Author: hongzhi-gao

.github/workflows/wheels.yml

@@ -63,7 +63,7 @@ jobs:
       - name: Install build tools
         run: |
           python -m pip install -U pip wheel
-          python -m pip install cibuildwheel==2.21.3
+          python -m pip install cibuildwheel==3.4.1
 
       - name: Pre-download virtualenv for cibuildwheel
         run: |
@@ -108,8 +108,8 @@ jobs:
           CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*"
           CIBW_SKIP: "pp* *-musllinux*"
 
-          CIBW_MANYLINUX_X86_64_IMAGE: "manylinux2014"
-          CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux2014"
+          CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28"
+          CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28"
 
           MACOSX_DEPLOYMENT_TARGET: "12.0"
 
@@ -222,7 +222,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -268,8 +268,13 @@ jobs:
           # Build wheel via Maven (no clean — keep C++ artifacts from previous job)
           chmod +x mvnw || true
           cd python
+          BLACK_VER_PROP=""
+          case "${{ matrix.python-version }}" in
+            3.9) BLACK_VER_PROP="-Dblack.version=25.11.0" ;;
+          esac
           ../mvnw package -DskipTests \
-            -Dspotless.check.skip=true -Dspotless.apply.skip=true
+            -Dspotless.check.skip=true -Dspotless.apply.skip=true \
+            ${BLACK_VER_PROP}
           ls -la dist/
 
       - name: Verify wheel

python/pyproject.toml

@@ -37,9 +37,11 @@ maintainers = [
 ]
 dependencies = [
     "numpy>=2.0.0,<3",
-    "pandas>=2.0",
+    "pandas>=2.0,<2.3; python_full_version < '3.14.0'",
+    "pandas>=2.3.3; python_full_version >= '3.14.0'",
     "pyarrow>=16.0,<18; python_version<'3.10'",
-    "pyarrow>=18.0,<20; python_version>='3.10'"
+    "pyarrow>=18.0,<20; python_version>='3.10' and python_version<'3.14'",
+    "pyarrow>=22.0,<24; python_version>='3.14'"
 ]
 
 [project.urls]

python/requirements.txt

@@ -18,9 +18,11 @@
 #
 
 cython==3.0.10
-black==26.3.1
+black==25.11.0; python_version < "3.10"
+black==26.3.1; python_version >= "3.10"
 numpy>=2.0.0,<3
-pandas==2.2.2
+pandas==2.2.2; python_full_version < "3.14.0"
+pandas>=2.3.3; python_full_version >= "3.14.0"
 setuptools==78.1.1
 wheel==0.46.2
 pyarrow>=8.0.0

python/tests/test_dataframe.py

@@ -51,6 +51,13 @@ def convert_to_nullable_types(df):
             df[col] = df[col].astype("Float32")
         elif dtype == "bool":
             df[col] = df[col].astype("boolean")
+        elif pd.api.types.is_object_dtype(df[col]):
+            non_null = df[col].dropna()
+            if len(non_null) and non_null.map(lambda x: isinstance(x, str)).all():
+                df[col] = df[col].astype("string")
+        elif pd.api.types.is_string_dtype(df[col]):
+            # NumPy/pandas str dtype (e.g. Py3.14) vs object: unify for Series.equals
+            df[col] = df[col].astype("string")
     return df
 
 
@@ -92,6 +99,7 @@ def test_write_dataframe_basic():
         df_sorted = convert_to_nullable_types(
             df.sort_values("time").reset_index(drop=True)
         )
+        df_read = convert_to_nullable_types(df_read)
         assert df_read.shape == (100, 4)
         assert df_read[TIME_COLUMN].equals(df_sorted["time"])
         assert df_read["device"].equals(df_sorted["device"])
@@ -128,6 +136,7 @@ def test_write_dataframe_with_index():
         df_read = df_read.sort_values(TIME_COLUMN).reset_index(drop=True)
         df_sorted = df.sort_index()
         df_sorted = convert_to_nullable_types(df_sorted.reset_index(drop=True))
+        df_read = convert_to_nullable_types(df_read)
         time_series = pd.Series(df.sort_index().index.values, dtype="Int64")
         assert df_read.shape == (50, 3)
         assert df_read[TIME_COLUMN].equals(time_series)
@@ -166,6 +175,7 @@ def test_write_dataframe_case_insensitive():
         df_sorted = convert_to_nullable_types(
             df.sort_values("Time").reset_index(drop=True)
         )
+        df_read = convert_to_nullable_types(df_read)
         assert df_read.shape == (30, 3)
         assert df_read[TIME_COLUMN].equals(df_sorted["Time"])
         assert df_read["device"].equals(df_sorted["Device"])
@@ -270,6 +280,7 @@ def test_write_dataframe_all_datatypes():
         df_sorted = convert_to_nullable_types(
             df.sort_values("time").reset_index(drop=True)
         )
+        df_read = convert_to_nullable_types(df_read)
         assert df_read.shape == (50, 11)
         assert df_read["bool_col"].equals(df_sorted["bool_col"])
         assert df_read["int32_col"].equals(df_sorted["int32_col"])
@@ -317,6 +328,7 @@ def test_write_dataframe_schema_time_column():
         df_sorted = convert_to_nullable_types(
             df.sort_values("time").reset_index(drop=True)
         )
+        df_read = convert_to_nullable_types(df_read)
         assert df_read.shape == (50, 3)
         assert df_read[TIME_COLUMN].equals(df_sorted[TIME_COLUMN])
         assert df_read["device"].equals(df_sorted["device"])
@@ -354,6 +366,7 @@ def test_write_dataframe_schema_time_and_dataframe_time():
         df_sorted = convert_to_nullable_types(
             df.sort_values("Time").rename(columns=str.lower).reset_index(drop=True)
         )
+        df_read = convert_to_nullable_types(df_read)
         assert df_read.shape == (30, 3)
         assert df_read["time"].equals(df_sorted["time"])
         assert df_read["device"].equals(df_sorted["device"])

python/tests/test_to_tsfile.py

@@ -40,6 +40,12 @@ def convert_to_nullable_types(df):
             df[col] = df[col].astype("Float32")
         elif dtype == "bool":
             df[col] = df[col].astype("boolean")
+        elif pd.api.types.is_object_dtype(df[col]):
+            non_null = df[col].dropna()
+            if len(non_null) and non_null.map(lambda x: isinstance(x, str)).all():
+                df[col] = df[col].astype("string")
+        elif pd.api.types.is_string_dtype(df[col]):
+            df[col] = df[col].astype("string")
     return df
 
 
@@ -65,6 +71,7 @@ def test_dataframe_to_tsfile_basic():
         df_sorted = convert_to_nullable_types(
             df.sort_values("time").reset_index(drop=True)
         )
+        df_read = convert_to_nullable_types(df_read)
 
         assert df_read.shape == (100, 4)
         assert df_read["time"].equals(df_sorted["time"])
@@ -109,6 +116,7 @@ def test_dataframe_to_tsfile_with_index():
 
         df_read = to_dataframe(tsfile_path, table_name="test_table")
         df_read = df_read.sort_values("time").reset_index(drop=True)
+        df_read = convert_to_nullable_types(df_read)
         time_expected = pd.Series(df.index.values, dtype="Int64")
         assert df_read.shape == (30, 3)
         assert df_read["time"].equals(time_expected)
@@ -146,6 +154,7 @@ def test_dataframe_to_tsfile_custom_time_column():
         df_sorted = convert_to_nullable_types(
             df.sort_values("timestamp").reset_index(drop=True)
         )
+        df_read = convert_to_nullable_types(df_read)
 
         assert df_read.shape == (30, 3)
         assert df_read["timestamp"].equals(df_sorted["timestamp"])
@@ -169,6 +178,7 @@ def test_dataframe_to_tsfile_case_insensitive_time():
         dataframe_to_tsfile(df, tsfile_path, table_name="test_table")
 
         df_read = to_dataframe(tsfile_path, table_name="test_table")
+        df_read = convert_to_nullable_types(df_read)
         assert df_read.shape == (20, 2)
         assert df_read["time"].equals(pd.Series([i for i in range(20)], dtype="Int64"))
     finally:
@@ -200,6 +210,7 @@ def test_dataframe_to_tsfile_with_tag_columns():
         df_sorted = convert_to_nullable_types(
             df.sort_values("time").reset_index(drop=True)
         )
+        df_read = convert_to_nullable_types(df_read)
 
         assert df_read.shape == (20, 4)
         assert df_read["device"].equals(df_sorted["device"])
@@ -242,6 +253,7 @@ def test_dataframe_to_tsfile_tag_time_unsorted():
         df_read = to_dataframe(tsfile_path, table_name="test_table")
         df_expected = df.sort_values(by=["device", "time"]).reset_index(drop=True)
         df_expected = convert_to_nullable_types(df_expected)
+        df_read = convert_to_nullable_types(df_read)
 
         assert df_read.shape == (10, 3)
         assert df_read["device"].equals(df_expected["device"])
@@ -281,6 +293,7 @@ def test_dataframe_to_tsfile_all_datatypes():
         df_sorted = convert_to_nullable_types(
             df.sort_values("time").reset_index(drop=True)
         )
+        df_read = convert_to_nullable_types(df_read)
 
         assert df_read.shape == (50, 11)
         assert df_read["bool_col"].equals(df_sorted["bool_col"])

python/tsfile/tsfile_table_writer.py

@@ -34,9 +34,19 @@ def validate_dataframe_for_tsfile(df: pd.DataFrame) -> None:
     seen = set()
     duplicates = []
     for c in columns:
-        if c is None or (isinstance(c, str) and len(c) == 0):
+        if isinstance(c, str):
+            if len(c) == 0:
+                raise ValueError("Column name cannot be None or empty")
+            lower = c.lower()
+        elif c is None:
             raise ValueError("Column name cannot be None or empty")
-        lower = c.lower()
+        else:
+            try:
+                if pd.isna(c):
+                    raise ValueError("Column name cannot be None or empty")
+            except TypeError:
+                pass
+            lower = str(c).lower()
         if lower in seen:
             duplicates.append(c)
         seen.add(lower)