You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by ff...@apache.org on 2026/10/08 14:47:32 UTC
(orc) branch main updated: ORC-2222: [C++] Support Java fixed-offset timezone IDs
This is an automated email from the ASF dual-hosted git repository.
ffacs pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/orc.git
The following commit(s) were added to refs/heads/main by this push:
new 2069271e0 ORC-2222: [C++] Support Java fixed-offset timezone IDs
2069271e0 is described below
commit 2069271e06c92197c66def9d4a0214753ac9a3b8
Author: ffacs <ff...@gmail.com>
AuthorDate: Thu Oct 8 22:47:16 2026 +0800
ORC-2222: [C++] Support Java fixed-offset timezone IDs
### What changes were proposed in this pull request?
Add support for Java fixed-offset timezone IDs (GMT±HH:MM and GMT±HH:MM:SS) in the C++ timezone resolver, using cached fixed-offset timezones without requiring TZDB files.
### Why are the changes needed?
The Java writer can record IDs such as GMT-00:00 or GMT+05:30 in the stripe’s writerTimezone. The C++ reader treats these IDs as timezone database filenames and fails when reading
timestamp columns because the corresponding files do not exist.
This change allows the C++ reader to resolve these offsets and read the timestamps correctly.
Fixes ORC-2222 and #2715.
### How was this patch tested?
Added tests covering:
• Positive and negative offsets, signed zero, minute and second precision, and invalid formats.
• Epoch calculation, time conversion, caching, and operation without TZDB files.
• Reading Java-written ORC files with four fixed-offset timezones, including pre-1970 timestamps and nanosecond precision.
Rebuilt C++ and ran make -C build test-out: all 763 core tests and 87 tool tests passed. Also verified that the original file attached to #2715 reads successfully.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-6)
Closes #2716 from ffacs/fix-java-fixed-offset-timezone.
Authored-by: ffacs <ff...@gmail.com>
Signed-off-by: ffacs <ff...@gmail.com>
---
c++/src/Timezone.cc | 65 +++++++++++++++++++++++++++++++++++
c++/test/TestTimezone.cc | 89 ++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 154 insertions(+)
diff --git a/c++/src/Timezone.cc b/c++/src/Timezone.cc
index b9499bd6d..4789fcb90 100644
--- a/c++/src/Timezone.cc
+++ b/c++/src/Timezone.cc
@@ -713,6 +713,65 @@ namespace orc {
return buffer;
}
+ // Java TimeZone IDs use GMT followed by an ISO-style offset, not the
+ // reversed sign used by POSIX TZ strings. ZoneOffset can also include seconds.
+ static bool parseFixedOffset(const std::string& zone, int64_t& offset) {
+ if ((zone.size() != 9 && zone.size() != 12) || zone.compare(0, 3, "GMT") != 0 ||
+ (zone[3] != '+' && zone[3] != '-')) {
+ return false;
+ }
+ int64_t parts[3] = {0, 0, 0};
+ for (size_t i = 0; i < (zone.size() == 9 ? 2 : 3); ++i) {
+ size_t start = 4 + i * 3;
+ if ((i != 0 && zone[start - 1] != ':') || zone[start] < '0' || zone[start] > '9' ||
+ zone[start + 1] < '0' || zone[start + 1] > '9') {
+ return false;
+ }
+ parts[i] = (zone[start] - '0') * 10 + zone[start + 1] - '0';
+ }
+ if (parts[0] > 23 || parts[1] > 59 || parts[2] > 59) {
+ return false;
+ }
+ offset = (parts[0] * 60 + parts[1]) * 60 + parts[2];
+ if (zone[3] == '-') {
+ offset = -offset;
+ }
+ return true;
+ }
+
+ class FixedOffsetTimezone : public Timezone {
+ public:
+ FixedOffsetTimezone(const std::string& name, int64_t offset) : variant_{offset, false, name} {}
+
+ const TimezoneVariant& getVariant(int64_t) const override {
+ return variant_;
+ }
+
+ int64_t getEpoch() const override {
+ // 2015-01-01 00:00:00 in this timezone, expressed as Unix seconds.
+ return 1420070400 - variant_.gmtOffset;
+ }
+
+ void print(std::ostream& out) const override {
+ out << variant_.toString() << "\n";
+ }
+
+ uint64_t getVersion() const override {
+ return 0; // No TZif file.
+ }
+
+ int64_t convertToUTC(int64_t clk) const override {
+ return clk + variant_.gmtOffset;
+ }
+
+ int64_t convertFromUTC(int64_t clk) const override {
+ return clk - variant_.gmtOffset;
+ }
+
+ private:
+ TimezoneVariant variant_;
+ };
+
class LazyTimezone : public Timezone {
private:
std::string filename_;
@@ -766,6 +825,12 @@ namespace orc {
if (itr != timezoneCache.end()) {
return *(itr->second).get();
}
+ int64_t offset;
+ if (parseFixedOffset(zone, offset)) {
+ auto timezone = std::make_shared<FixedOffsetTimezone>(zone, offset);
+ timezoneCache[filename] = timezone;
+ return *timezone;
+ }
auto it = TZ_ALIASES.find(zone);
if (it == TZ_ALIASES.end()) {
timezoneCache[filename] = std::make_shared<LazyTimezone>(filename);
diff --git a/c++/test/TestTimezone.cc b/c++/test/TestTimezone.cc
index ceb4b7b6a..c9715b795 100644
--- a/c++/test/TestTimezone.cc
+++ b/c++/test/TestTimezone.cc
@@ -17,13 +17,16 @@
*/
#include "Adaptor.hh"
+#include "MemoryInputStream.hh"
#include "Timezone.hh"
+#include "orc/OrcFile.hh"
#include "wrap/gmock.h"
#include "wrap/gtest-wrapper.h"
#include <algorithm>
#include <cstdlib>
#include <iostream>
+#include <utility>
#include <vector>
namespace orc {
@@ -363,6 +366,91 @@ namespace orc {
EXPECT_EQ("EST", getVariantFromZone(*ny1, "1974-10-27 06:00:00"));
}
+ TEST(TestTimezone, testFixedOffsets) {
+ const std::pair<std::string, int64_t> cases[] = {
+ {"GMT-00:00", 0}, {"GMT+00:00", 0}, {"GMT+05:30", 19800},
+ {"GMT-03:30", -12600}, {"GMT+05:45", 20700}, {"GMT+00:10", 600},
+ {"GMT+23:59", 86340}, {"GMT-23:59", -86340}, {"GMT+05:30:15", 19815},
+ {"GMT-03:30:15", -12615}, {"GMT+00:00:01", 1}, {"GMT-00:00:59", -59}};
+ for (const auto& entry : cases) {
+ SCOPED_TRACE(entry.first);
+ const auto& zone = getTimezoneByName(entry.first);
+ EXPECT_EQ(&zone, &getTimezoneByName(entry.first));
+ EXPECT_EQ(1420070400 - entry.second, zone.getEpoch());
+ for (int64_t time : {-2208988800LL, 0LL, 1420070400LL, 4102444800LL}) {
+ const auto& variant = zone.getVariant(time);
+ EXPECT_EQ(entry.second, variant.gmtOffset);
+ EXPECT_FALSE(variant.isDst);
+ EXPECT_EQ(entry.first, variant.name);
+ EXPECT_EQ(time + entry.second, zone.convertToUTC(time));
+ EXPECT_EQ(time - entry.second, zone.convertFromUTC(time));
+ }
+ }
+ }
+
+ TEST(TestTimezone, testReadJavaFixedOffsets) {
+ // Written by the Java ORC 2.1.2 writer with OpenJDK 22.0.2. Each file contains
+ // struct<ts:timestamp>, compression NONE, rowIndexStride 0, and these values:
+ // 2001-11-12 18:31:01.123456789, 1960-01-02 03:04:05, and
+ // 2040-07-01 12:00:00.999999999. TimeZone.setDefault is called before creating
+ // the writer; the seconds offset uses TimeZone.getTimeZone(ZoneOffset.of(...)).
+ const std::pair<std::string, std::string> files[] = {
+ {"GMT-00:00",
+ "T1JDdgIxaPE1zuYRtV/tFIB4AgA63mioAAAAAAAB3NZP+AoGCAEQARgOCgYIBRABGBESAggAEgIIAhoJR01U"
+ "LTAwOjAwChAKBAgAUAAKCAgASgBQAFgfCAMQRRoKCAMQABgfICMoAyIJCAwSAQEaAnRzIgIICTADOgQIAFAA"
+ "OggIAEoAUABYH0AASABYAWIFMi4xLjIIPhAAIgIADCgSMAmC9AMDT1JDEw=="},
+ {"GMT+05:30",
+ "T1JDdgIxaPE1zuYRtV/tFIB4AgA63mioAAAAAAAB3NZP+AoGCAEQARgOCgYIBRABGBESAggAEgIIAhoJR01U"
+ "KzA1OjMwChAKBAgAUAAKCAgASgBQAFgfCAMQRRoKCAMQABgfICMoAyIJCAwSAQEaAnRzIgIICTADOgQIAFAA"
+ "OggIAEoAUABYH0AASABYAWIFMi4xLjIIPhAAIgIADCgSMAmC9AMDT1JDEw=="},
+ {"GMT-03:30",
+ "T1JDdgIxaPE1zuYRtV/tFIB4AgA63mioAAAAAAAB3NZP+AoGCAEQARgOCgYIBRABGBESAggAEgIIAhoJR01U"
+ "LTAzOjMwChAKBAgAUAAKCAgASgBQAFgfCAMQRRoKCAMQABgfICMoAyIJCAwSAQEaAnRzIgIICTADOgQIAFAA"
+ "OggIAEoAUABYH0AASABYAWIFMi4xLjIIPhAAIgIADCgSMAmC9AMDT1JDEw=="},
+ {"GMT+05:30:15",
+ "T1JDdgIxaPE1zuYRtV/tFIB4AgA63mioAAAAAAAB3NZP+AoGCAEQARgOCgYIBRABGBESAggAEgIIAhoMR01U"
+ "KzA1OjMwOjE1ChAKBAgAUAAKCAgASgBQAFgfCAMQSBoKCAMQABgfICYoAyIJCAwSAQEaAnRzIgIICTADOgQI"
+ "AFAAOggIAEoAUABYH0AASABYAWIFMi4xLjIIPhAAIgIADCgSMAmC9AMDT1JDEw=="},
+ };
+ const int64_t seconds[] = {1005589861, -315521755, 2224756800};
+ const int64_t nanos[] = {123456789, 0, 999999999};
+ for (const auto& file : files) {
+ SCOPED_TRACE(file.first);
+ auto bytes = decodeBase64(file.second);
+ auto reader = createReader(std::make_unique<MemoryInputStream>(
+ reinterpret_cast<const char*>(bytes.data()), bytes.size()),
+ ReaderOptions());
+ // Check conversion to both UTC and another fixed offset. In both cases
+ // the reader must preserve the original wall-clock timestamp.
+ for (const auto& readerZone : {"GMT+00:00", "GMT-03:30"}) {
+ SCOPED_TRACE(readerZone);
+ RowReaderOptions options;
+ options.setTimezoneName(readerZone);
+ auto rows = reader->createRowReader(options);
+ auto batch = rows->createRowBatch(10);
+ ASSERT_TRUE(rows->next(*batch));
+ ASSERT_EQ(3, batch->numElements);
+ const auto& timestamps = dynamic_cast<const TimestampVectorBatch&>(
+ *dynamic_cast<const StructVectorBatch&>(*batch).fields[0]);
+ const int64_t readerOffset = std::string(readerZone) == "GMT-03:30" ? -12600 : 0;
+ for (size_t i = 0; i < 3; ++i) {
+ EXPECT_EQ(seconds[i] - readerOffset, timestamps.data[i]);
+ EXPECT_EQ(nanos[i], timestamps.nanoseconds[i]);
+ }
+ EXPECT_FALSE(rows->next(*batch));
+ }
+ }
+ }
+
+ TEST(TestTimezone, testInvalidFixedOffsets) {
+ for (const auto& name :
+ {"GMT+24:00", "GMT-00:60", "GMT+00:00:60", "GMT+05:30x", "GMT+05:30:15x", "GMT+0a:00",
+ "GMT+00/00", "GMT*05:30", "GMT+5:30", "GMT+05:3", "gmt+05:30", "GMT+"}) {
+ SCOPED_TRACE(name);
+ EXPECT_THROW(getTimezoneByName(name).getEpoch(), TimezoneError);
+ }
+ }
+
TEST(TestTimezone, testGMTv1) {
const char GMT[] =
("VFppZgAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAQAAAA"
@@ -437,6 +525,7 @@ namespace orc {
testing::ThrowsMessage<TimezoneError>(testing::HasSubstr(
"Time zone file /path/to/wrong/tzdb/America/Los_Angeles does not exist."
" Please install IANA time zone database and set TZDIR env.")));
+ EXPECT_EQ(1420050600, getTimezoneByName("GMT+05:30").getEpoch());
if (!tzDirBackup.empty()) {
ASSERT_TRUE(setEnv("TZDIR", tzDirBackup.c_str()));
} else {