You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@arrow.apache.org by "kou (via GitHub)" <gi...@apache.org> on 2025/11/22 13:34:09 UTC

Re: [PR] GH-48204: [C++][Parquet] Fix Column Reader & Writer logic to enable Parquet DB support on s390x [arrow]

kou commented on code in PR #48205:
URL: https://github.com/apache/arrow/pull/48205#discussion_r2553142118


##########
cpp/src/parquet/column_reader.cc:
##########
@@ -132,7 +134,11 @@ int LevelDecoder::SetData(Encoding::type encoding, int16_t max_level,
             "Number of buffered values too large (corrupt data page?)");
       }
       num_bytes = static_cast<int32_t>(bit_util::BytesForBits(num_bits));
+#if ARROW_LITTLE_ENDIAN
       if (num_bytes < 0 || num_bytes > data_size - 4) {
+#else
+      if (num_bytes < 0 || num_bytes > data_size) {
+#endif

Review Comment:
   @pitrou You added `- 4` in #6848. Do you think that we need `- 4` with big endian too?



##########
cpp/src/parquet/column_writer.h:
##########
@@ -260,13 +261,21 @@ constexpr int64_t kJulianEpochOffsetDays = INT64_C(2440588);
 template <int64_t UnitPerDay, int64_t NanosecondsPerUnit>
 inline void ArrowTimestampToImpalaTimestamp(const int64_t time, Int96* impala_timestamp) {
   int64_t julian_days = (time / UnitPerDay) + kJulianEpochOffsetDays;
+#if ARROW_LITTLE_ENDIAN
   (*impala_timestamp).value[2] = (uint32_t)julian_days;
+#endif
 
   int64_t last_day_units = time % UnitPerDay;
   auto last_day_nanos = last_day_units * NanosecondsPerUnit;
+#if ARROW_LITTLE_ENDIAN
   // impala_timestamp will be unaligned every other entry so do memcpy instead
   // of assign and reinterpret cast to avoid undefined behavior.
   std::memcpy(impala_timestamp, &last_day_nanos, sizeof(int64_t));
+#else
+  (*impala_timestamp).value[0] = static_cast<uint32_t>(last_day_nanos);
+  (*impala_timestamp).value[1] = static_cast<uint32_t>(last_day_nanos >> 32);

Review Comment:
   Can we use the following instead of `#if`?
   
   ```cpp
   auto last_day_nanos = last_day_units * NanosecondsPerUnit;
   auto last_day_nanos_little_endian = ::arrow::bit_util::ToLittleEndian(last_day_nanos);
   std::memcpy(impala_timestamp, &last_day_nanos_little_endian, sizeof(int64_t));
   ```



##########
cpp/src/parquet/column_writer.cc:
##########
@@ -2621,6 +2641,38 @@ struct SerializeFunctor<::parquet::FLBAType, ::arrow::HalfFloatType> {
     return FLBA{reinterpret_cast<const uint8_t*>(value_ptr)};
   }
 };
+#else
+template <>
+struct SerializeFunctor<::parquet::FLBAType, ::arrow::HalfFloatType> {
+  Status Serialize(const ::arrow::HalfFloatArray& array, ArrowWriteContext*, FLBA* out) {
+    const uint16_t* values = array.raw_values();
+    const int64_t length = array.length();
+
+    // Allocate buffer for little-endian converted values
+    converted_values_.resize(length);
+
+    if (array.null_count() == 0) {
+      for (int64_t i = 0; i < length; ++i) {
+        converted_values_[i] = ::arrow::bit_util::ToLittleEndian(values[i]);
+        out[i] = FLBA{reinterpret_cast<const uint8_t*>(&converted_values_[i])};
+      }
+    } else {
+      for (int64_t i = 0; i < length; ++i) {
+        if (array.IsValid(i)) {
+          converted_values_[i] = ::arrow::bit_util::ToLittleEndian(values[i]);
+          out[i] = FLBA{reinterpret_cast<const uint8_t*>(&converted_values_[i])};
+        } else {
+          out[i] = FLBA{};
+        }
+      }
+    }
+    return Status::OK();
+  }
+
+ private:
+  std::vector<uint16_t> converted_values_;
+};
+#endif

Review Comment:
   Could you share implementation as much as possible something like:
   
   ```cpp
   template <>
   struct SerializeFunctor<::parquet::FLBAType, ::arrow::HalfFloatType> {
     Status Serialize(const ::arrow::HalfFloatArray& array, ArrowWriteContext*, FLBA* out) {
   #if ARROW_LITTLE_ENDIAN
       return SerializeLittleEndianValues(array.raw_values(), out);
   #else
       const uint16_t* values = array.raw_values();
       const int64_t length = array.length();
       converted_values_.resize(length);
       for (int64_t i = 0; i < length; ++i) {
         // We don't need IsValid() here. Non valid values are just ignored in SerializeLittleEndianValues().
         converted_values_[i] = ::arrow::bit_util::ToLittleEndian(values[i]);
       }
       return SerializeLittleEndianValues(converted_values_.data(), out);
   #endif
     }
   
    private:
     Status SerializeLittleEndianValues(const uint16_t* values, FLBA* out) {
       if (array.null_count() == 0) {
         for (int64_t i = 0; i < array.length(); ++i) {
           out[i] = ToFLBA(&values[i]);
         }
       } else {
         for (int64_t i = 0; i < array.length(); ++i) {
           out[i] = array.IsValid(i) ? ToFLBA(&values[i]) : FLBA{};
         }
       }
       return Status::OK();
     }
   
     FLBA ToFLBA(const uint16_t* value_ptr) const {
       return FLBA{reinterpret_cast<const uint8_t*>(value_ptr)};
     }
   
   #if !ARROW_LITTLE_ENDIAN
     std::vector<uint16_t> converted_values_;  
   #endif
   };
   ```



##########
cpp/src/parquet/column_writer.h:
##########
@@ -260,13 +261,21 @@ constexpr int64_t kJulianEpochOffsetDays = INT64_C(2440588);
 template <int64_t UnitPerDay, int64_t NanosecondsPerUnit>
 inline void ArrowTimestampToImpalaTimestamp(const int64_t time, Int96* impala_timestamp) {
   int64_t julian_days = (time / UnitPerDay) + kJulianEpochOffsetDays;
+#if ARROW_LITTLE_ENDIAN
   (*impala_timestamp).value[2] = (uint32_t)julian_days;
+#endif

Review Comment:
   Do we need this `#if`?
   
   It seems that the below `(*impala_timestamp).value[2] = static_cast<uint32_t>(julian_days);` does the same thing.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org