You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@arrow.apache.org by GitBox <gi...@apache.org> on 2021/06/14 15:11:26 UTC
[GitHub] [arrow] jorisvandenbossche commented on a change in pull request #10457: ARROW-12980: [C++] Kernels to extract datetime components should be timezone aware
jorisvandenbossche commented on a change in pull request #10457:
URL: https://github.com/apache/arrow/pull/10457#discussion_r651034869
##########
File path: python/pyarrow/tests/test_compute.py
##########
@@ -1212,6 +1212,80 @@ def test_strptime():
assert got == expected
+def _check_datetime_components(ts):
+ tsa = [pa.array(ts)]
+ subseconds = ((ts.microsecond * 10**3 + ts.nanosecond) *
+ 10**-9).values.round(9)
+ iso_calendar_fields = [
+ pa.field('iso_year', pa.int64()),
+ pa.field('iso_week', pa.int64()),
+ pa.field('iso_day_of_week', pa.int64())
+ ]
+ iso_calendar = pa.StructArray.from_arrays([
+ ts.isocalendar()["year"].astype(int),
+ ts.isocalendar()["week"].astype(int),
+ ts.isocalendar()["day"].astype(int)],
+ fields=iso_calendar_fields)
+
+ assert pc.call_function("year", tsa).equals(pa.array(ts.year))
Review comment:
You can directly use `pc.year(..)` instead of `pc.call_function("year", ..)` (and the same for the others below)
##########
File path: python/pyarrow/tests/test_compute.py
##########
@@ -1212,6 +1212,80 @@ def test_strptime():
assert got == expected
+def _check_datetime_components(ts):
+ tsa = [pa.array(ts)]
+ subseconds = ((ts.microsecond * 10**3 + ts.nanosecond) *
+ 10**-9).values.round(9)
+ iso_calendar_fields = [
+ pa.field('iso_year', pa.int64()),
+ pa.field('iso_week', pa.int64()),
+ pa.field('iso_day_of_week', pa.int64())
+ ]
+ iso_calendar = pa.StructArray.from_arrays([
+ ts.isocalendar()["year"].astype(int),
+ ts.isocalendar()["week"].astype(int),
+ ts.isocalendar()["day"].astype(int)],
+ fields=iso_calendar_fields)
+
+ assert pc.call_function("year", tsa).equals(pa.array(ts.year))
+ assert pc.call_function("month", tsa).equals(pa.array(ts.month))
+ assert pc.call_function("day", tsa).equals(pa.array(ts.day))
+ assert pc.call_function("day_of_week", tsa).equals(
+ pa.array(ts.day_of_week))
+ assert pc.call_function("day_of_year", tsa).equals(
+ pa.array(ts.day_of_year))
+ assert pc.call_function("iso_year", tsa).equals(
+ pa.array(ts.isocalendar()["year"].astype(int)))
+ assert pc.call_function("iso_week", tsa).equals(
+ pa.array(ts.isocalendar()["week"].astype(int)))
+ assert pc.call_function("iso_calendar", tsa).equals(iso_calendar)
+ assert pc.call_function("quarter", tsa).equals(pa.array(ts.quarter))
+ assert pc.call_function("hour", tsa).equals(pa.array(ts.hour))
+ assert pc.call_function("minute", tsa).equals(pa.array(ts.minute))
+ assert pc.call_function("second", tsa).equals(pa.array(ts.second.values))
+ assert pc.call_function("millisecond", tsa).equals(
+ pa.array(ts.microsecond // 10**3))
+ assert pc.call_function("microsecond", tsa).equals(
+ pa.array(ts.microsecond % 10**3))
+ assert pc.call_function("nanosecond", tsa).equals(pa.array(ts.nanosecond))
+ assert pc.call_function("subsecond", tsa).equals(pa.array(subseconds))
+
+
+@pytest.mark.pandas
+def test_extract_datetime_components():
+ import pandas as pd
+
+ # TODO: see https://github.com/pandas-dev/pandas/issues/41834
+ # "1899-01-01T00:59:20.001001001"
+ timestamps = ["1970-01-01T00:00:59.123456789",
+ "2000-02-29T23:23:23.999999999",
+ "2033-05-18T03:33:20.000000000",
+ "2020-01-01T01:05:05.001",
+ "2019-12-31T02:10:10.002",
+ "2019-12-30T03:15:15.003",
+ "2009-12-31T04:20:20.004132",
+ "2010-01-01T05:25:25.005321",
+ "2010-01-03T06:30:30.006163",
+ "2010-01-04T07:35:35",
+ "2006-01-01T08:40:40",
+ "2005-12-31T09:45:45",
+ "2008-12-28",
+ "2008-12-29",
+ "2012-01-01 01:02:03"]
+ timezones = ["US/Central", "Pacific/Marquesas", "Asia/Kolkata",
+ "Etc/GMT-4", "Etc/GMT+4", "Pacific/Marquesas",
+ "Australia/Broken_Hill"]
+
+ # Test timezone naive timestamp array
+ ts = pd.to_datetime(timestamps)
+ _check_datetime_components(ts)
+
+ # Test timezone aware timestamp array
+ for timezone in timezones:
+ ts = pd.to_datetime(timestamps).tz_localize("UTC").tz_convert(timezone)
Review comment:
```suggestion
ts = pd.to_datetime(timestamps).tz_localize(timezone)
```
Otherwise the "local" timestamps might change, and some of the corner cases might be shifted to another day.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org