You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2018/11/22 14:56:18 UTC
[arrow] branch master updated: ARROW-3738: [C++] Parse ISO8601-like
timestamps in CSV columns
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new d06d0d0 ARROW-3738: [C++] Parse ISO8601-like timestamps in CSV columns
d06d0d0 is described below
commit d06d0d0b435ec13b759164c18be52217a3c4e418
Author: Antoine Pitrou <an...@python.org>
AuthorDate: Thu Nov 22 09:56:09 2018 -0500
ARROW-3738: [C++] Parse ISO8601-like timestamps in CSV columns
Second granularity is allowed (we might want to add support for fractions of seconds, e.g. in the "YYYY-MM-DD[T ]hh:mm:ss.ssssss" format).
Timestamp conversion also participates in CSV type inference, since it's unlikely to produce false positives (e.g. a semantically "string" column that would be entirely made of valid timestamp strings).
Author: Antoine Pitrou <an...@python.org>
Closes #2952 from pitrou/ARROW-3738-csv-timestamps and squashes the following commits:
005a6e3f7 <Antoine Pitrou> ARROW-3738: Parse ISO8601-like timestamps in CSV columns
---
LICENSE.txt | 2 +-
cpp/CMakeLists.txt | 2 +-
cpp/build-support/clang_format_exclusions.txt | 2 +-
cpp/build-support/lint_cpp_cli.py | 2 +-
cpp/src/arrow/csv/column-builder.cc | 10 +-
cpp/src/arrow/csv/converter.cc | 36 ++++++
cpp/src/arrow/csv/csv-column-builder-test.cc | 35 ++++-
cpp/src/arrow/csv/csv-converter-test.cc | 23 ++++
cpp/src/arrow/test-util.h | 52 +++++---
cpp/src/arrow/util/CMakeLists.txt | 1 +
cpp/src/{gandiva/precompiled => arrow/util}/date.h | 113 ++++++++--------
cpp/src/arrow/util/parsing-util-test.cc | 96 +++++++++++++-
cpp/src/arrow/util/parsing.h | 144 +++++++++++++++++++--
cpp/src/gandiva/precompiled/epoch_time_point.h | 2 +-
cpp/src/gandiva/precompiled/time_test.cc | 1 -
cpp/src/gandiva/to_date_holder.cc | 3 +-
python/doc/source/csv.rst | 9 +-
python/pyarrow/tests/test_csv.py | 13 ++
18 files changed, 445 insertions(+), 101 deletions(-)
diff --git a/LICENSE.txt b/LICENSE.txt
index 2651a13..2e6ce20 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -717,7 +717,7 @@ See the License for the specific language governing permissions and
limitations under the License.
--------------------------------------------------------------------------------
-The file cpp/src/gandiva/precompiled/date.h has the following license (MIT)
+The file cpp/src/arrow/util/date.h has the following license (MIT)
The MIT License (MIT)
Copyright (c) 2015, 2016, 2017 Howard Hinnant
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index cfa71be..8f3ed08 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -346,10 +346,10 @@ if (UNIX)
(item MATCHES "xxhash.h") OR
(item MATCHES "xxhash.cc") OR
(item MATCHES "config.h") OR
+ (item MATCHES "util/date.h") OR
(item MATCHES "util/string_view/") OR
(item MATCHES "util/variant") OR
(item MATCHES "zmalloc.h") OR
- (item MATCHES "gandiva/precompiled/date.h") OR
(item MATCHES "ae.h")))
LIST(APPEND FILTERED_LINT_FILES ${item})
ENDIF()
diff --git a/cpp/build-support/clang_format_exclusions.txt b/cpp/build-support/clang_format_exclusions.txt
index 1aeecfa..c04523a 100644
--- a/cpp/build-support/clang_format_exclusions.txt
+++ b/cpp/build-support/clang_format_exclusions.txt
@@ -4,6 +4,7 @@
*pyarrow_lib.h
*python/config.h
*python/platform.h
+*util/date.h
*util/string_view/*
*util/variant.h
*util/variant/*
@@ -11,4 +12,3 @@
*xxhash.cc
*xxhash.h
*RcppExports.cpp*
-*gandiva/precompiled/date.h
diff --git a/cpp/build-support/lint_cpp_cli.py b/cpp/build-support/lint_cpp_cli.py
index 993ea2f..2333d7f 100644
--- a/cpp/build-support/lint_cpp_cli.py
+++ b/cpp/build-support/lint_cpp_cli.py
@@ -70,12 +70,12 @@ def lint_file(path):
EXCLUSIONS = [
'arrow/python/iterators.h',
+ 'arrow/util/date.h',
'arrow/util/macros.h',
'arrow/util/parallel.h',
'arrow/util/string_view/string_view.hpp',
'gandiva/cache.h',
'gandiva/jni',
- 'gandiva/precompiled/date.h',
'test',
'internal'
]
diff --git a/cpp/src/arrow/csv/column-builder.cc b/cpp/src/arrow/csv/column-builder.cc
index ba2685f..28cbad4 100644
--- a/cpp/src/arrow/csv/column-builder.cc
+++ b/cpp/src/arrow/csv/column-builder.cc
@@ -167,7 +167,7 @@ class InferringColumnBuilder : public ColumnBuilder {
std::shared_ptr<Converter> converter_;
// Current inference status
- enum class InferKind { Null, Integer, Real, Text, Binary };
+ enum class InferKind { Null, Integer, Real, Timestamp, Text, Binary };
std::shared_ptr<DataType> infer_type_;
InferKind infer_kind_;
@@ -191,6 +191,9 @@ Status InferringColumnBuilder::LoosenType() {
infer_kind_ = InferKind::Integer;
break;
case InferKind::Integer:
+ infer_kind_ = InferKind::Timestamp;
+ break;
+ case InferKind::Timestamp:
infer_kind_ = InferKind::Real;
break;
case InferKind::Real:
@@ -217,6 +220,11 @@ Status InferringColumnBuilder::UpdateType() {
infer_type_ = int64();
can_loosen_type_ = true;
break;
+ case InferKind::Timestamp:
+ // We don't support parsing second fractions for now
+ infer_type_ = timestamp(TimeUnit::SECOND);
+ can_loosen_type_ = true;
+ break;
case InferKind::Real:
infer_type_ = float64();
can_loosen_type_ = true;
diff --git a/cpp/src/arrow/csv/converter.cc b/cpp/src/arrow/csv/converter.cc
index b959711..0f056ce 100644
--- a/cpp/src/arrow/csv/converter.cc
+++ b/cpp/src/arrow/csv/converter.cc
@@ -329,6 +329,41 @@ Status NumericConverter<T>::Convert(const BlockParser& parser, int32_t col_index
return Status::OK();
}
+/////////////////////////////////////////////////////////////////////////
+// Concrete Converter for timestamps
+
+class TimestampConverter : public ConcreteConverter {
+ public:
+ using ConcreteConverter::ConcreteConverter;
+
+ Status Convert(const BlockParser& parser, int32_t col_index,
+ std::shared_ptr<Array>* out) override {
+ using value_type = TimestampType::c_type;
+
+ TimestampBuilder builder(type_, pool_);
+ StringConverter<TimestampType> converter(type_);
+
+ auto visit = [&](const uint8_t* data, uint32_t size, bool quoted) -> Status {
+ value_type value;
+ if (IsNull(data, size, quoted)) {
+ builder.UnsafeAppendNull();
+ return Status::OK();
+ }
+ if (ARROW_PREDICT_FALSE(
+ !converter(reinterpret_cast<const char*>(data), size, &value))) {
+ return GenericConversionError(type_, data, size);
+ }
+ builder.UnsafeAppend(value);
+ return Status::OK();
+ };
+ RETURN_NOT_OK(builder.Resize(parser.num_rows()));
+ RETURN_NOT_OK(parser.VisitColumn(col_index, visit));
+ RETURN_NOT_OK(builder.Finish(out));
+
+ return Status::OK();
+ }
+};
+
} // namespace
/////////////////////////////////////////////////////////////////////////
@@ -361,6 +396,7 @@ Status Converter::Make(const std::shared_ptr<DataType>& type,
CONVERTER_CASE(Type::FLOAT, NumericConverter<FloatType>)
CONVERTER_CASE(Type::DOUBLE, NumericConverter<DoubleType>)
CONVERTER_CASE(Type::BOOL, NumericConverter<BooleanType>)
+ CONVERTER_CASE(Type::TIMESTAMP, TimestampConverter)
CONVERTER_CASE(Type::BINARY, (VarSizeBinaryConverter<BinaryType, false>))
CONVERTER_CASE(Type::FIXED_SIZE_BINARY, FixedSizeBinaryConverter)
diff --git a/cpp/src/arrow/csv/csv-column-builder-test.cc b/cpp/src/arrow/csv/csv-column-builder-test.cc
index 7488ad8..a039183 100644
--- a/cpp/src/arrow/csv/csv-column-builder-test.cc
+++ b/cpp/src/arrow/csv/csv-column-builder-test.cc
@@ -51,7 +51,7 @@ void AssertBuilding(const std::shared_ptr<ColumnBuilder>& builder,
}
//////////////////////////////////////////////////////////////////////////
-// Test functions begin here
+// Tests for fixed-type column builder
TEST(ColumnBuilder, Empty) {
auto tg = TaskGroup::MakeSerial();
@@ -123,6 +123,9 @@ TEST(ColumnBuilder, MultipleChunksParallel) {
AssertChunkedEqual(*actual, *expected);
}
+//////////////////////////////////////////////////////////////////////////
+// Tests for type-inferring column builder
+
TEST(InferringColumnBuilder, Empty) {
auto tg = TaskGroup::MakeSerial();
std::shared_ptr<ColumnBuilder> builder;
@@ -214,6 +217,36 @@ TEST(InferringColumnBuilder, MultipleChunkReal) {
AssertChunkedEqual(*expected, *actual);
}
+TEST(InferringColumnBuilder, SingleChunkTimestamp) {
+ auto tg = TaskGroup::MakeSerial();
+ std::shared_ptr<ColumnBuilder> builder;
+ ASSERT_OK(ColumnBuilder::Make(0, ConvertOptions::Defaults(), tg, &builder));
+
+ std::shared_ptr<ChunkedArray> actual;
+ AssertBuilding(builder, {{"", "1970-01-01", "2018-11-13 17:11:10"}}, &actual);
+
+ std::shared_ptr<ChunkedArray> expected;
+ ChunkedArrayFromVector<TimestampType>(timestamp(TimeUnit::SECOND),
+ {{false, true, true}}, {{0, 0, 1542129070}},
+ &expected);
+ AssertChunkedEqual(*expected, *actual);
+}
+
+TEST(InferringColumnBuilder, MultipleChunkTimestamp) {
+ auto tg = TaskGroup::MakeSerial();
+ std::shared_ptr<ColumnBuilder> builder;
+ ASSERT_OK(ColumnBuilder::Make(0, ConvertOptions::Defaults(), tg, &builder));
+
+ std::shared_ptr<ChunkedArray> actual;
+ AssertBuilding(builder, {{""}, {"1970-01-01"}, {"2018-11-13 17:11:10"}}, &actual);
+
+ std::shared_ptr<ChunkedArray> expected;
+ ChunkedArrayFromVector<TimestampType>(timestamp(TimeUnit::SECOND),
+ {{false}, {true}, {true}},
+ {{0}, {0}, {1542129070}}, &expected);
+ AssertChunkedEqual(*expected, *actual);
+}
+
TEST(InferringColumnBuilder, SingleChunkString) {
auto tg = TaskGroup::MakeSerial();
std::shared_ptr<ColumnBuilder> builder;
diff --git a/cpp/src/arrow/csv/csv-converter-test.cc b/cpp/src/arrow/csv/csv-converter-test.cc
index 024a54c..2534541 100644
--- a/cpp/src/arrow/csv/csv-converter-test.cc
+++ b/cpp/src/arrow/csv/csv-converter-test.cc
@@ -220,6 +220,29 @@ TEST(BooleanConversion, Nulls) {
{{true, true}, {false, true}});
}
+TEST(TimestampConversion, Basics) {
+ auto type = timestamp(TimeUnit::SECOND);
+
+ AssertConversion<TimestampType, int64_t>(
+ type, {"1970-01-01\n2000-02-29\n3989-07-14\n1900-02-28\n"},
+ {{0, 951782400, 63730281600LL, -2203977600LL}});
+ AssertConversion<TimestampType, int64_t>(type,
+ {"2018-11-13 17:11:10\n1900-02-28 12:34:56\n"},
+ {{1542129070, -2203932304LL}});
+
+ type = timestamp(TimeUnit::NANO);
+ AssertConversion<TimestampType, int64_t>(
+ type, {"1970-01-01\n2000-02-29\n1900-02-28\n"},
+ {{0, 951782400000000000LL, -2203977600000000000LL}});
+}
+
+TEST(TimestampConversion, Nulls) {
+ auto type = timestamp(TimeUnit::MILLI);
+ AssertConversion<TimestampType, int64_t>(type, {"1970-01-01 00:01:00,,N/A\n"},
+ {{60000}, {0}, {0}},
+ {{true}, {false}, {false}});
+}
+
TEST(DecimalConversion, NotImplemented) {
std::shared_ptr<Converter> converter;
ASSERT_RAISES(NotImplemented,
diff --git a/cpp/src/arrow/test-util.h b/cpp/src/arrow/test-util.h
index 296aa88..7cab89e 100644
--- a/cpp/src/arrow/test-util.h
+++ b/cpp/src/arrow/test-util.h
@@ -222,6 +222,8 @@ Status MakeRandomBuffer(int64_t length, MemoryPool* pool,
return Status::OK();
}
+// ArrayFromVector: construct an Array from vectors of C values
+
template <typename TYPE, typename C_TYPE = typename TYPE::c_type>
void ArrayFromVector(const std::shared_ptr<DataType>& type,
const std::vector<bool>& is_valid, const std::vector<C_TYPE>& values,
@@ -261,56 +263,68 @@ void ArrayFromVector(const std::shared_ptr<DataType>& type,
ASSERT_OK(builder.Finish(out));
}
+// Overloads without a DataType argument, for parameterless types
+
template <typename TYPE, typename C_TYPE = typename TYPE::c_type>
void ArrayFromVector(const std::vector<bool>& is_valid, const std::vector<C_TYPE>& values,
std::shared_ptr<Array>* out) {
- typename TypeTraits<TYPE>::BuilderType builder;
- DCHECK_EQ(is_valid.size(), values.size());
- for (size_t i = 0; i < values.size(); ++i) {
- if (is_valid[i]) {
- ASSERT_OK(builder.Append(values[i]));
- } else {
- ASSERT_OK(builder.AppendNull());
- }
- }
- ASSERT_OK(builder.Finish(out));
+ auto type = TypeTraits<TYPE>::type_singleton();
+ ArrayFromVector<TYPE, C_TYPE>(type, is_valid, values, out);
}
template <typename TYPE, typename C_TYPE = typename TYPE::c_type>
void ArrayFromVector(const std::vector<C_TYPE>& values, std::shared_ptr<Array>* out) {
- typename TypeTraits<TYPE>::BuilderType builder;
- for (auto& value : values) {
- ASSERT_OK(builder.Append(value));
- }
- ASSERT_OK(builder.Finish(out));
+ auto type = TypeTraits<TYPE>::type_singleton();
+ ArrayFromVector<TYPE, C_TYPE>(type, values, out);
}
+// ChunkedArrayFromVector: construct a ChunkedArray from vectors of C values
+
template <typename TYPE, typename C_TYPE = typename TYPE::c_type>
-void ChunkedArrayFromVector(const std::vector<std::vector<bool>>& is_valid,
+void ChunkedArrayFromVector(const std::shared_ptr<DataType>& type,
+ const std::vector<std::vector<bool>>& is_valid,
const std::vector<std::vector<C_TYPE>>& values,
std::shared_ptr<ChunkedArray>* out) {
ArrayVector chunks;
DCHECK_EQ(is_valid.size(), values.size());
for (size_t i = 0; i < values.size(); ++i) {
std::shared_ptr<Array> array;
- ArrayFromVector<TYPE, C_TYPE>(is_valid[i], values[i], &array);
+ ArrayFromVector<TYPE, C_TYPE>(type, is_valid[i], values[i], &array);
chunks.push_back(array);
}
*out = std::make_shared<ChunkedArray>(chunks);
}
template <typename TYPE, typename C_TYPE = typename TYPE::c_type>
-void ChunkedArrayFromVector(const std::vector<std::vector<C_TYPE>>& values,
+void ChunkedArrayFromVector(const std::shared_ptr<DataType>& type,
+ const std::vector<std::vector<C_TYPE>>& values,
std::shared_ptr<ChunkedArray>* out) {
ArrayVector chunks;
for (size_t i = 0; i < values.size(); ++i) {
std::shared_ptr<Array> array;
- ArrayFromVector<TYPE, C_TYPE>(values[i], &array);
+ ArrayFromVector<TYPE, C_TYPE>(type, values[i], &array);
chunks.push_back(array);
}
*out = std::make_shared<ChunkedArray>(chunks);
}
+// Overloads without a DataType argument, for parameterless types
+
+template <typename TYPE, typename C_TYPE = typename TYPE::c_type>
+void ChunkedArrayFromVector(const std::vector<std::vector<bool>>& is_valid,
+ const std::vector<std::vector<C_TYPE>>& values,
+ std::shared_ptr<ChunkedArray>* out) {
+ auto type = TypeTraits<TYPE>::type_singleton();
+ ChunkedArrayFromVector<TYPE, C_TYPE>(type, is_valid, values, out);
+}
+
+template <typename TYPE, typename C_TYPE = typename TYPE::c_type>
+void ChunkedArrayFromVector(const std::vector<std::vector<C_TYPE>>& values,
+ std::shared_ptr<ChunkedArray>* out) {
+ auto type = TypeTraits<TYPE>::type_singleton();
+ ChunkedArrayFromVector<TYPE, C_TYPE>(type, values, out);
+}
+
template <class T, class Builder>
Status MakeArray(const std::vector<uint8_t>& valid_bytes, const std::vector<T>& values,
int64_t size, Builder* builder, std::shared_ptr<Array>* out) {
diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt
index 9f962f4..5d813ec 100644
--- a/cpp/src/arrow/util/CMakeLists.txt
+++ b/cpp/src/arrow/util/CMakeLists.txt
@@ -34,6 +34,7 @@ install(FILES
compression_zlib.h
compression_zstd.h
cpu-info.h
+ date.h
decimal.h
hash-util.h
hash.h
diff --git a/cpp/src/gandiva/precompiled/date.h b/cpp/src/arrow/util/date.h
similarity index 99%
rename from cpp/src/gandiva/precompiled/date.h
rename to cpp/src/arrow/util/date.h
index 3c6f874..aa76488 100644
--- a/cpp/src/gandiva/precompiled/date.h
+++ b/cpp/src/arrow/util/date.h
@@ -1,3 +1,5 @@
+// Vendored from https://github.com/HowardHinnant/date/
+
#ifndef DATE_H
#define DATE_H
@@ -285,9 +287,9 @@ class day {
explicit CONSTCD11 day(unsigned d) NOEXCEPT;
CONSTCD14 day& operator++() NOEXCEPT;
- CONSTCD14 day operator++(int) NOEXCEPT;
+ CONSTCD14 day operator++(int) NOEXCEPT;
CONSTCD14 day& operator--() NOEXCEPT;
- CONSTCD14 day operator--(int) NOEXCEPT;
+ CONSTCD14 day operator--(int) NOEXCEPT;
CONSTCD14 day& operator+=(const days& d) NOEXCEPT;
CONSTCD14 day& operator-=(const days& d) NOEXCEPT;
@@ -322,9 +324,9 @@ class month {
explicit CONSTCD11 month(unsigned m) NOEXCEPT;
CONSTCD14 month& operator++() NOEXCEPT;
- CONSTCD14 month operator++(int) NOEXCEPT;
+ CONSTCD14 month operator++(int) NOEXCEPT;
CONSTCD14 month& operator--() NOEXCEPT;
- CONSTCD14 month operator--(int) NOEXCEPT;
+ CONSTCD14 month operator--(int) NOEXCEPT;
CONSTCD14 month& operator+=(const months& m) NOEXCEPT;
CONSTCD14 month& operator-=(const months& m) NOEXCEPT;
@@ -359,9 +361,9 @@ class year {
explicit CONSTCD11 year(int y) NOEXCEPT;
CONSTCD14 year& operator++() NOEXCEPT;
- CONSTCD14 year operator++(int) NOEXCEPT;
+ CONSTCD14 year operator++(int) NOEXCEPT;
CONSTCD14 year& operator--() NOEXCEPT;
- CONSTCD14 year operator--(int) NOEXCEPT;
+ CONSTCD14 year operator--(int) NOEXCEPT;
CONSTCD14 year& operator+=(const years& y) NOEXCEPT;
CONSTCD14 year& operator-=(const years& y) NOEXCEPT;
@@ -406,9 +408,9 @@ class weekday {
CONSTCD11 explicit weekday(const local_days& dp) NOEXCEPT;
CONSTCD14 weekday& operator++() NOEXCEPT;
- CONSTCD14 weekday operator++(int) NOEXCEPT;
+ CONSTCD14 weekday operator++(int) NOEXCEPT;
CONSTCD14 weekday& operator--() NOEXCEPT;
- CONSTCD14 weekday operator--(int) NOEXCEPT;
+ CONSTCD14 weekday operator--(int) NOEXCEPT;
CONSTCD14 weekday& operator+=(const days& d) NOEXCEPT;
CONSTCD14 weekday& operator-=(const days& d) NOEXCEPT;
@@ -1056,7 +1058,7 @@ struct no_overflow {
-((std::intmax_t(1) << (sizeof(std::intmax_t) * CHAR_BIT - 1)) + 1);
template <std::intmax_t Xp, std::intmax_t Yp, bool overflow>
- struct mul { // overflow == false
+ struct mul { // overflow == false
static const std::intmax_t value = Xp * Yp;
};
@@ -1206,7 +1208,7 @@ CONSTCD14 inline day& day::operator++() NOEXCEPT {
++d_;
return *this;
}
-CONSTCD14 inline day day:: operator++(int) NOEXCEPT {
+CONSTCD14 inline day day::operator++(int) NOEXCEPT {
auto tmp(*this);
++(*this);
return tmp;
@@ -1215,7 +1217,7 @@ CONSTCD14 inline day& day::operator--() NOEXCEPT {
--d_;
return *this;
}
-CONSTCD14 inline day day:: operator--(int) NOEXCEPT {
+CONSTCD14 inline day day::operator--(int) NOEXCEPT {
auto tmp(*this);
--(*this);
return tmp;
@@ -1289,7 +1291,7 @@ CONSTCD14 inline month& month::operator++() NOEXCEPT {
*this += months{1};
return *this;
}
-CONSTCD14 inline month month:: operator++(int) NOEXCEPT {
+CONSTCD14 inline month month::operator++(int) NOEXCEPT {
auto tmp(*this);
++(*this);
return tmp;
@@ -1298,7 +1300,7 @@ CONSTCD14 inline month& month::operator--() NOEXCEPT {
*this -= months{1};
return *this;
}
-CONSTCD14 inline month month:: operator--(int) NOEXCEPT {
+CONSTCD14 inline month month::operator--(int) NOEXCEPT {
auto tmp(*this);
--(*this);
return tmp;
@@ -1379,7 +1381,7 @@ CONSTCD14 inline year& year::operator++() NOEXCEPT {
++y_;
return *this;
}
-CONSTCD14 inline year year:: operator++(int) NOEXCEPT {
+CONSTCD14 inline year year::operator++(int) NOEXCEPT {
auto tmp(*this);
++(*this);
return tmp;
@@ -1388,7 +1390,7 @@ CONSTCD14 inline year& year::operator--() NOEXCEPT {
--y_;
return *this;
}
-CONSTCD14 inline year year:: operator--(int) NOEXCEPT {
+CONSTCD14 inline year year::operator--(int) NOEXCEPT {
auto tmp(*this);
--(*this);
return tmp;
@@ -1498,7 +1500,7 @@ CONSTCD14 inline weekday& weekday::operator++() NOEXCEPT {
*this += days{1};
return *this;
}
-CONSTCD14 inline weekday weekday:: operator++(int) NOEXCEPT {
+CONSTCD14 inline weekday weekday::operator++(int) NOEXCEPT {
auto tmp(*this);
++(*this);
return tmp;
@@ -1507,7 +1509,7 @@ CONSTCD14 inline weekday& weekday::operator--() NOEXCEPT {
*this -= days{1};
return *this;
}
-CONSTCD14 inline weekday weekday:: operator--(int) NOEXCEPT {
+CONSTCD14 inline weekday weekday::operator--(int) NOEXCEPT {
auto tmp(*this);
--(*this);
return tmp;
@@ -2518,7 +2520,7 @@ inline bool year_month_weekday_last::ok() const NOEXCEPT {
CONSTCD14
inline days year_month_weekday_last::to_days() const NOEXCEPT {
auto const d = sys_days(y_ / m_ / last);
- return (d - (date::weekday {d} - wdl_.weekday())).time_since_epoch();
+ return (d - (date::weekday{d} - wdl_.weekday())).time_since_epoch();
}
CONSTCD11
@@ -3140,7 +3142,7 @@ class time_of_day_storage<std::chrono::duration<Rep, Period>, detail::classify::
}
CONSTCD11 bool in_conventional_range() const NOEXCEPT {
- return base::in_conventional_range() && m_ < std::chrono::hours {1};
+ return base::in_conventional_range() && m_ < std::chrono::hours{1};
}
template <class CharT, class Traits>
@@ -3219,7 +3221,7 @@ class time_of_day_storage<std::chrono::duration<Rep, Period>, detail::classify::
}
CONSTCD11 bool in_conventional_range() const NOEXCEPT {
- return base::in_conventional_range() && m_ < std::chrono::hours {1} &&
+ return base::in_conventional_range() && m_ < std::chrono::hours{1} &&
s_.in_conventional_range();
}
@@ -3314,7 +3316,7 @@ class time_of_day_storage<std::chrono::duration<Rep, Period>, detail::classify::
}
CONSTCD11 bool in_conventional_range() const NOEXCEPT {
- return base::in_conventional_range() && m_ < std::chrono::hours {1} &&
+ return base::in_conventional_range() && m_ < std::chrono::hours{1} &&
s_.in_conventional_range();
}
@@ -3617,14 +3619,14 @@ std::basic_ostream<CharT, Traits>& to_stream(std::basic_ostream<CharT, Traits>&
const fields<Duration>& fds,
const std::string* abbrev,
const std::chrono::seconds* offset_sec) {
- using std::chrono::duration_cast;
- using std::chrono::duration;
- using std::use_facet;
- using std::chrono::hours;
- using std::chrono::minutes;
using detail::save_ostream;
using std::ios;
using std::time_put;
+ using std::use_facet;
+ using std::chrono::duration;
+ using std::chrono::duration_cast;
+ using std::chrono::hours;
+ using std::chrono::minutes;
date::detail::save_ostream<CharT, Traits> ss(os);
os.fill(' ');
os.flags(std::ios::skipws | std::ios::dec);
@@ -3715,11 +3717,11 @@ std::basic_ostream<CharT, Traits>& to_stream(std::basic_ostream<CharT, Traits>&
os << month_names().first[extract_month(os, fds) - 1 + 12] << ' ';
auto d = static_cast<int>(static_cast<unsigned>(fds.ymd.day()));
if (d < 10) {
- os << ' ';
+ os << ' ';
}
os << d << ' ' << make_time(duration_cast<seconds>(fds.tod.to_duration()))
<< ' ' << fds.ymd.year();
- } else { // *fmt == 'x'
+ } else { // *fmt == 'x'
auto const& ymd = fds.ymd;
save_ostream<CharT, Traits> _(os);
os.fill('0');
@@ -4032,10 +4034,10 @@ std::basic_ostream<CharT, Traits>& to_stream(std::basic_ostream<CharT, Traits>&
tm.tm_hour = static_cast<int>(fds.tod.hours().count());
facet.put(os, os, os.fill(), &tm, begin(f), end(f));
#else
- if (fds.tod.hours() < hours {12}) {
+ if (fds.tod.hours() < hours{12}) {
os << ampm_names().first[0];
} else {
- os << ampm_names().first[1];
+ os << ampm_names().first[1];
}
#endif
} else {
@@ -4044,7 +4046,7 @@ std::basic_ostream<CharT, Traits>& to_stream(std::basic_ostream<CharT, Traits>&
modified = CharT{};
command = nullptr;
} else {
- os << *fmt;
+ os << *fmt;
}
break;
case 'r':
@@ -4069,10 +4071,10 @@ std::basic_ostream<CharT, Traits>& to_stream(std::basic_ostream<CharT, Traits>&
os.width(2);
os << tod.seconds().count() << CharT{' '};
tod.make24();
- if (tod.hours() < hours {12}) {
+ if (tod.hours() < hours{12}) {
os << ampm_names().first[0];
} else {
- os << ampm_names().first[1];
+ os << ampm_names().first[1];
}
#endif
} else {
@@ -4081,16 +4083,22 @@ std::basic_ostream<CharT, Traits>& to_stream(std::basic_ostream<CharT, Traits>&
modified = CharT{};
command = nullptr;
} else {
- os << *fmt;
+ os << *fmt;
}
break;
case 'R':
if (command) {
if (modified == CharT{}) {
- if (!fds.has_tod) { os.setstate(std::ios::failbit); }
- if (fds.tod.hours() < hours{10}) { os << CharT{'0'}; }
+ if (!fds.has_tod) {
+ os.setstate(std::ios::failbit);
+ }
+ if (fds.tod.hours() < hours{10}) {
+ os << CharT{'0'};
+ }
os << fds.tod.hours().count() << CharT{':'};
- if (fds.tod.minutes() < minutes{10}) { os << CharT{'0'}; }
+ if (fds.tod.minutes() < minutes{10}) {
+ os << CharT{'0'};
+ }
os << fds.tod.minutes().count();
} else {
os << CharT{'%'} << modified << *fmt;
@@ -4128,7 +4136,7 @@ std::basic_ostream<CharT, Traits>& to_stream(std::basic_ostream<CharT, Traits>&
modified = CharT{};
command = nullptr;
} else {
- os << *fmt;
+ os << *fmt;
}
break;
case 't':
@@ -4141,7 +4149,7 @@ std::basic_ostream<CharT, Traits>& to_stream(std::basic_ostream<CharT, Traits>&
}
command = nullptr;
} else {
- os << *fmt;
+ os << *fmt;
}
break;
case 'T':
@@ -4155,15 +4163,14 @@ std::basic_ostream<CharT, Traits>& to_stream(std::basic_ostream<CharT, Traits>&
}
command = nullptr;
} else {
- os << *fmt;
+ os << *fmt;
}
break;
case 'u':
if (command) {
if (modified == CharT{'E'}) {
os << CharT{'%'} << modified << *fmt;
- }
- else {
+ } else {
auto wd = extract_weekday(os, fds);
#if !ONLY_C_LOCALE
if (modified == CharT{})
@@ -4182,7 +4189,7 @@ std::basic_ostream<CharT, Traits>& to_stream(std::basic_ostream<CharT, Traits>&
modified = CharT{};
command = nullptr;
} else {
- os << *fmt;
+ os << *fmt;
}
break;
case 'U':
@@ -4221,7 +4228,7 @@ std::basic_ostream<CharT, Traits>& to_stream(std::basic_ostream<CharT, Traits>&
modified = CharT{};
command = nullptr;
} else {
- os << *fmt;
+ os << *fmt;
}
break;
case 'V':
@@ -4263,7 +4270,7 @@ std::basic_ostream<CharT, Traits>& to_stream(std::basic_ostream<CharT, Traits>&
modified = CharT{};
command = nullptr;
} else {
- os << *fmt;
+ os << *fmt;
}
break;
case 'w':
@@ -4291,7 +4298,7 @@ std::basic_ostream<CharT, Traits>& to_stream(std::basic_ostream<CharT, Traits>&
modified = CharT{};
command = nullptr;
} else {
- os << *fmt;
+ os << *fmt;
}
break;
case 'W':
@@ -6364,29 +6371,29 @@ msl(std::ratio<N, D>) {
template <class CharT>
CONSTCD11 inline string_literal<CharT, 2> msl(std::atto) NOEXCEPT {
- return string_literal<CharT, 2> {'a'};
+ return string_literal<CharT, 2>{'a'};
}
template <class CharT>
CONSTCD11 inline string_literal<CharT, 2> msl(std::femto) NOEXCEPT {
- return string_literal<CharT, 2> {'f'};
+ return string_literal<CharT, 2>{'f'};
}
template <class CharT>
CONSTCD11 inline string_literal<CharT, 2> msl(std::pico) NOEXCEPT {
- return string_literal<CharT, 2> {'p'};
+ return string_literal<CharT, 2>{'p'};
}
template <class CharT>
CONSTCD11 inline string_literal<CharT, 2> msl(std::nano) NOEXCEPT {
- return string_literal<CharT, 2> {'n'};
+ return string_literal<CharT, 2>{'n'};
}
template <class CharT>
CONSTCD11 inline typename std::enable_if<std::is_same<CharT, char>::value,
string_literal<char, 3>>::type
msl(std::micro) NOEXCEPT {
- return string_literal<char, 3> {'\xC2', '\xB5'};
+ return string_literal<char, 3>{'\xC2', '\xB5'};
}
template <class CharT>
@@ -6398,12 +6405,12 @@ msl(std::micro) NOEXCEPT {
template <class CharT>
CONSTCD11 inline string_literal<CharT, 2> msl(std::milli) NOEXCEPT {
- return string_literal<CharT, 2> {'m'};
+ return string_literal<CharT, 2>{'m'};
}
template <class CharT>
CONSTCD11 inline string_literal<CharT, 2> msl(std::centi) NOEXCEPT {
- return string_literal<CharT, 2> {'c'};
+ return string_literal<CharT, 2>{'c'};
}
template <class CharT>
diff --git a/cpp/src/arrow/util/parsing-util-test.cc b/cpp/src/arrow/util/parsing-util-test.cc
index dd7718d..ff0621a 100644
--- a/cpp/src/arrow/util/parsing-util-test.cc
+++ b/cpp/src/arrow/util/parsing-util-test.cc
@@ -33,7 +33,7 @@ void AssertConversion(ConverterType& converter, const std::string& s, C_TYPE exp
typename ConverterType::value_type out;
ASSERT_TRUE(converter(s.data(), s.length(), &out))
<< "Conversion failed for '" << s << "' (expected to return " << expected << ")";
- ASSERT_EQ(out, expected);
+ ASSERT_EQ(out, expected) << "Conversion failed for '" << s << "'";
}
template <typename ConverterType>
@@ -264,4 +264,98 @@ TEST(StringConversion, ToUInt64) {
AssertConversionFails(converter, "e");
}
+TEST(StringConversion, ToTimestamp1) {
+ {
+ StringConverter<TimestampType> converter(timestamp(TimeUnit::SECOND));
+
+ AssertConversion(converter, "1970-01-01", 0);
+ AssertConversion(converter, "1989-07-14", 616377600);
+ AssertConversion(converter, "2000-02-29", 951782400);
+ AssertConversion(converter, "3989-07-14", 63730281600LL);
+ AssertConversion(converter, "1900-02-28", -2203977600LL);
+
+ AssertConversionFails(converter, "");
+ AssertConversionFails(converter, "1970");
+ AssertConversionFails(converter, "19700101");
+ AssertConversionFails(converter, "1970/01/01");
+ AssertConversionFails(converter, "1970-01-01 ");
+ AssertConversionFails(converter, "1970-01-01Z");
+
+ // Invalid dates
+ AssertConversionFails(converter, "1970-00-01");
+ AssertConversionFails(converter, "1970-13-01");
+ AssertConversionFails(converter, "1970-01-32");
+ AssertConversionFails(converter, "1970-02-29");
+ AssertConversionFails(converter, "2100-02-29");
+ }
+ {
+ StringConverter<TimestampType> converter(timestamp(TimeUnit::MILLI));
+
+ AssertConversion(converter, "1970-01-01", 0);
+ AssertConversion(converter, "1989-07-14", 616377600000LL);
+ AssertConversion(converter, "3989-07-14", 63730281600000LL);
+ AssertConversion(converter, "1900-02-28", -2203977600000LL);
+ }
+ {
+ StringConverter<TimestampType> converter(timestamp(TimeUnit::MICRO));
+
+ AssertConversion(converter, "1970-01-01", 0);
+ AssertConversion(converter, "1989-07-14", 616377600000000LL);
+ AssertConversion(converter, "3989-07-14", 63730281600000000LL);
+ AssertConversion(converter, "1900-02-28", -2203977600000000LL);
+ }
+ {
+ StringConverter<TimestampType> converter(timestamp(TimeUnit::NANO));
+
+ AssertConversion(converter, "1970-01-01", 0);
+ AssertConversion(converter, "1989-07-14", 616377600000000000LL);
+ AssertConversion(converter, "2018-11-13", 1542067200000000000LL);
+ AssertConversion(converter, "1900-02-28", -2203977600000000000LL);
+ }
+}
+
+TEST(StringConversion, ToTimestamp2) {
+ {
+ StringConverter<TimestampType> converter(timestamp(TimeUnit::SECOND));
+
+ AssertConversion(converter, "1970-01-01 00:00:00", 0);
+ AssertConversion(converter, "2018-11-13 17:11:10", 1542129070);
+ AssertConversion(converter, "2018-11-13T17:11:10", 1542129070);
+ AssertConversion(converter, "2018-11-13 17:11:10Z", 1542129070);
+ AssertConversion(converter, "2018-11-13T17:11:10Z", 1542129070);
+ AssertConversion(converter, "1900-02-28 12:34:56", -2203932304LL);
+
+ // Invalid dates
+ AssertConversionFails(converter, "1970-02-29 00:00:00");
+ AssertConversionFails(converter, "2100-02-29 00:00:00");
+ // Invalid times
+ AssertConversionFails(converter, "1970-01-01 24:00:00");
+ AssertConversionFails(converter, "1970-01-01 00:60:00");
+ AssertConversionFails(converter, "1970-01-01 00:00:60");
+ }
+ {
+ StringConverter<TimestampType> converter(timestamp(TimeUnit::MILLI));
+
+ AssertConversion(converter, "2018-11-13 17:11:10", 1542129070000LL);
+ AssertConversion(converter, "2018-11-13T17:11:10Z", 1542129070000LL);
+ AssertConversion(converter, "3989-07-14T11:22:33Z", 63730322553000LL);
+ AssertConversion(converter, "1900-02-28 12:34:56", -2203932304000LL);
+ }
+ {
+ StringConverter<TimestampType> converter(timestamp(TimeUnit::MICRO));
+
+ AssertConversion(converter, "2018-11-13 17:11:10", 1542129070000000LL);
+ AssertConversion(converter, "2018-11-13T17:11:10Z", 1542129070000000LL);
+ AssertConversion(converter, "3989-07-14T11:22:33Z", 63730322553000000LL);
+ AssertConversion(converter, "1900-02-28 12:34:56", -2203932304000000LL);
+ }
+ {
+ StringConverter<TimestampType> converter(timestamp(TimeUnit::NANO));
+
+ AssertConversion(converter, "2018-11-13 17:11:10", 1542129070000000000LL);
+ AssertConversion(converter, "2018-11-13T17:11:10Z", 1542129070000000000LL);
+ AssertConversion(converter, "1900-02-28 12:34:56", -2203932304000000000LL);
+ }
+}
+
} // namespace arrow
diff --git a/cpp/src/arrow/util/parsing.h b/cpp/src/arrow/util/parsing.h
index 04aff1e..aa1f820 100644
--- a/cpp/src/arrow/util/parsing.h
+++ b/cpp/src/arrow/util/parsing.h
@@ -20,8 +20,11 @@
#ifndef ARROW_UTIL_PARSING_H
#define ARROW_UTIL_PARSING_H
+#include <cassert>
+#include <chrono>
#include <limits>
#include <locale>
+#include <memory>
#include <sstream>
#include <string>
#include <type_traits>
@@ -30,6 +33,8 @@
#include "arrow/type.h"
#include "arrow/type_traits.h"
+#include "arrow/util/checked_cast.h"
+#include "arrow/util/date.h"
namespace arrow {
namespace internal {
@@ -146,16 +151,18 @@ class StringConverter<DoubleType> : public StringToFloatConverterMixin<DoubleTyp
namespace detail {
-#define PARSE_UNSIGNED_ITERATION(C_TYPE) \
- if (length > 0) { \
- uint8_t digit = static_cast<uint8_t>(*s++ - '0'); \
- result = static_cast<C_TYPE>(result * 10U); \
- length--; \
- if (ARROW_PREDICT_FALSE(digit > 9U)) { \
- /* Non-digit */ \
- return false; \
- } \
- result = static_cast<C_TYPE>(result + digit); \
+inline uint8_t ParseDecimalDigit(char c) { return static_cast<uint8_t>(c - '0'); }
+
+#define PARSE_UNSIGNED_ITERATION(C_TYPE) \
+ if (length > 0) { \
+ uint8_t digit = ParseDecimalDigit(*s++); \
+ result = static_cast<C_TYPE>(result * 10U); \
+ length--; \
+ if (ARROW_PREDICT_FALSE(digit > 9U)) { \
+ /* Non-digit */ \
+ return false; \
+ } \
+ result = static_cast<C_TYPE>(result + digit); \
}
#define PARSE_UNSIGNED_ITERATION_LAST(C_TYPE) \
@@ -164,7 +171,7 @@ namespace detail {
/* Overflow */ \
return false; \
} \
- uint8_t digit = static_cast<uint8_t>(*s++ - '0'); \
+ uint8_t digit = ParseDecimalDigit(*s++); \
result = static_cast<C_TYPE>(result * 10U); \
C_TYPE new_result = static_cast<C_TYPE>(result + digit); \
if (ARROW_PREDICT_FALSE(--length > 0)) { \
@@ -351,6 +358,121 @@ class StringConverter<Int32Type> : public StringToSignedIntConverterMixin<Int32T
template <>
class StringConverter<Int64Type> : public StringToSignedIntConverterMixin<Int64Type> {};
+template <>
+class StringConverter<TimestampType> {
+ public:
+ using value_type = TimestampType::c_type;
+
+ explicit StringConverter(const std::shared_ptr<DataType>& type)
+ : unit_(checked_cast<TimestampType*>(type.get())->unit()) {}
+
+ bool operator()(const char* s, size_t length, value_type* out) {
+ // We allow the following formats:
+ // - "YYYY-MM-DD"
+ // - "YYYY-MM-DD[ T]hh:mm:ss"
+ // - "YYYY-MM-DD[ T]hh:mm:ssZ"
+ // UTC is always assumed, and the DataType's timezone is ignored.
+ date::year_month_day ymd;
+ if (ARROW_PREDICT_FALSE(length < 10)) {
+ return false;
+ }
+ if (length == 10) {
+ if (ARROW_PREDICT_FALSE(!ParseYYYY_MM_DD(s, &ymd))) {
+ return false;
+ }
+ return ConvertTimePoint(date::sys_days(ymd), out);
+ }
+ if (ARROW_PREDICT_FALSE(s[10] != ' ') && ARROW_PREDICT_FALSE(s[10] != 'T')) {
+ return false;
+ }
+ if (s[length - 1] == 'Z') {
+ --length;
+ }
+ if (length == 19) {
+ if (ARROW_PREDICT_FALSE(!ParseYYYY_MM_DD(s, &ymd))) {
+ return false;
+ }
+ std::chrono::duration<value_type> seconds;
+ if (ARROW_PREDICT_FALSE(!ParseHH_MM_SS(s + 11, &seconds))) {
+ return false;
+ }
+ return ConvertTimePoint(date::sys_days(ymd) + seconds, out);
+ }
+ return false;
+ }
+
+ protected:
+ template <class TimePoint>
+ bool ConvertTimePoint(TimePoint tp, value_type* out) {
+ auto duration = tp.time_since_epoch();
+ switch (unit_) {
+ case TimeUnit::SECOND:
+ *out = std::chrono::duration_cast<std::chrono::seconds>(duration).count();
+ return true;
+ case TimeUnit::MILLI:
+ *out = std::chrono::duration_cast<std::chrono::milliseconds>(duration).count();
+ return true;
+ case TimeUnit::MICRO:
+ *out = std::chrono::duration_cast<std::chrono::microseconds>(duration).count();
+ return true;
+ case TimeUnit::NANO:
+ *out = std::chrono::duration_cast<std::chrono::nanoseconds>(duration).count();
+ return true;
+ }
+ // Unreachable
+ assert(0);
+ return true;
+ }
+
+ bool ParseYYYY_MM_DD(const char* s, date::year_month_day* out) {
+ uint16_t year;
+ uint8_t month, day;
+ if (ARROW_PREDICT_FALSE(s[4] != '-') || ARROW_PREDICT_FALSE(s[7] != '-')) {
+ return false;
+ }
+ if (ARROW_PREDICT_FALSE(!detail::ParseUnsigned(s + 0, 4, &year))) {
+ return false;
+ }
+ if (ARROW_PREDICT_FALSE(!detail::ParseUnsigned(s + 5, 2, &month))) {
+ return false;
+ }
+ if (ARROW_PREDICT_FALSE(!detail::ParseUnsigned(s + 8, 2, &day))) {
+ return false;
+ }
+ *out = {date::year{year}, date::month{month}, date::day{day}};
+ return out->ok();
+ }
+
+ bool ParseHH_MM_SS(const char* s, std::chrono::duration<value_type>* out) {
+ uint8_t hours, minutes, seconds;
+ if (ARROW_PREDICT_FALSE(s[2] != ':') || ARROW_PREDICT_FALSE(s[5] != ':')) {
+ return false;
+ }
+ if (ARROW_PREDICT_FALSE(!detail::ParseUnsigned(s + 0, 2, &hours))) {
+ return false;
+ }
+ if (ARROW_PREDICT_FALSE(!detail::ParseUnsigned(s + 3, 2, &minutes))) {
+ return false;
+ }
+ if (ARROW_PREDICT_FALSE(!detail::ParseUnsigned(s + 6, 2, &seconds))) {
+ return false;
+ }
+ if (ARROW_PREDICT_FALSE(hours >= 24)) {
+ return false;
+ }
+ if (ARROW_PREDICT_FALSE(minutes >= 60)) {
+ return false;
+ }
+ if (ARROW_PREDICT_FALSE(seconds >= 60)) {
+ return false;
+ }
+ *out = std::chrono::duration<value_type>(3600U * hours + 60U * minutes + seconds);
+ return true;
+ }
+
+ const TimeUnit::type unit_;
+};
+
} // namespace internal
} // namespace arrow
diff --git a/cpp/src/gandiva/precompiled/epoch_time_point.h b/cpp/src/gandiva/precompiled/epoch_time_point.h
index aff72b4..dc6340d 100644
--- a/cpp/src/gandiva/precompiled/epoch_time_point.h
+++ b/cpp/src/gandiva/precompiled/epoch_time_point.h
@@ -19,7 +19,7 @@
#define GANDIVA_EPOCH_TIME_POINT_H
// TODO(wesm): IR compilation does not have any include directories set
-#include "./date.h"
+#include "../../arrow/util/date.h"
// A point of time measured in millis since epoch.
class EpochTimePoint {
diff --git a/cpp/src/gandiva/precompiled/time_test.cc b/cpp/src/gandiva/precompiled/time_test.cc
index fb60b24..b8ee4dc 100644
--- a/cpp/src/gandiva/precompiled/time_test.cc
+++ b/cpp/src/gandiva/precompiled/time_test.cc
@@ -19,7 +19,6 @@
#include <gtest/gtest.h>
#include "../execution_context.h"
-#include "gandiva/precompiled/date.h"
#include "gandiva/precompiled/types.h"
namespace gandiva {
diff --git a/cpp/src/gandiva/to_date_holder.cc b/cpp/src/gandiva/to_date_holder.cc
index 054e25f..9c85622 100644
--- a/cpp/src/gandiva/to_date_holder.cc
+++ b/cpp/src/gandiva/to_date_holder.cc
@@ -18,10 +18,11 @@
#include <algorithm>
#include <string>
+#include "arrow/util/date.h"
+
#include "gandiva/date_utils.h"
#include "gandiva/execution_context.h"
#include "gandiva/node.h"
-#include "gandiva/precompiled/date.h"
#include "gandiva/to_date_holder.h"
namespace gandiva {
diff --git a/python/doc/source/csv.rst b/python/doc/source/csv.rst
index 3fff8a8..f1bcea9 100644
--- a/python/doc/source/csv.rst
+++ b/python/doc/source/csv.rst
@@ -29,7 +29,7 @@ The features currently offered are the following:
such as ``my_data.csv.gz``)
* fetching column names from the first row in the CSV file
* column-wise type inference and conversion to one of ``null``, ``int64``,
- ``float64``, ``string`` or ``binary`` data
+ ``float64``, ``timestamp[s]``, ``string`` or ``binary`` data
* detecting various spellings of null values such as ``NaN`` or ``#N/A``
Usage
@@ -75,13 +75,6 @@ Customized conversion
To alter how CSV data is converted to Arrow types and data, you should create
a :class:`ConvertOptions` instance and pass it to :func:`read_csv`.
-Limitations
------------
-
-Arrow is not able to detect or convert other data types (such as dates
-and times) than the five mentioned above. It is also not possible to
-choose the data types of columns explicitly.
-
Performance
-----------
diff --git a/python/pyarrow/tests/test_csv.py b/python/pyarrow/tests/test_csv.py
index 7f67a54..115595b 100644
--- a/python/pyarrow/tests/test_csv.py
+++ b/python/pyarrow/tests/test_csv.py
@@ -16,6 +16,7 @@
# under the License.
import bz2
+from datetime import datetime
import gzip
import io
import itertools
@@ -220,6 +221,18 @@ class BaseTestCSVRead:
'e': [b"3", b"nan", b"\xff"],
}
+ def test_simple_timestamps(self):
+ # Infer a timestamp column
+ rows = b"a,b\n1970,1970-01-01\n1989,1989-07-14\n"
+ table = self.read_bytes(rows)
+ schema = pa.schema([('a', pa.int64()),
+ ('b', pa.timestamp('s'))])
+ assert table.schema == schema
+ assert table.to_pydict() == {
+ 'a': [1970, 1989],
+ 'b': [datetime(1970, 1, 1), datetime(1989, 7, 14)],
+ }
+
def test_column_types(self):
# Ask for specific column types in ConvertOptions
opts = ConvertOptions(column_types={'b': 'float32',