You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/05/31 17:46:30 UTC
[01/14] arrow git commit: [maven-release-plugin] prepare release
apache-arrow-0.4.0 [Forced Update!]
Repository: arrow
Updated Branches:
refs/heads/master 5899800f5 -> 51b6bf253 (forced update)
[maven-release-plugin] prepare release apache-arrow-0.4.0
Change-Id: I976a1bb24dde7f6b7b9fc2208b4f4995b1023756
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/a8f8ba0c
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/a8f8ba0c
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/a8f8ba0c
Branch: refs/heads/master
Commit: a8f8ba0cbcf5f596f042e90b7a208e7a0c3925b7
Parents: ff72951
Author: Wes McKinney <we...@twosigma.com>
Authored: Fri May 19 11:16:28 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Fri May 19 11:16:28 2017 -0400
----------------------------------------------------------------------
java/format/pom.xml | 2 +-
java/memory/pom.xml | 2 +-
java/pom.xml | 4 ++--
java/tools/pom.xml | 2 +-
java/vector/pom.xml | 2 +-
5 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/a8f8ba0c/java/format/pom.xml
----------------------------------------------------------------------
diff --git a/java/format/pom.xml b/java/format/pom.xml
index 4ef7482..6efdf93 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
<parent>
<artifactId>arrow-java-root</artifactId>
<groupId>org.apache.arrow</groupId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0</version>
</parent>
<artifactId>arrow-format</artifactId>
http://git-wip-us.apache.org/repos/asf/arrow/blob/a8f8ba0c/java/memory/pom.xml
----------------------------------------------------------------------
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index e6d9900..bba043c 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
<parent>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0</version>
</parent>
<artifactId>arrow-memory</artifactId>
<name>Arrow Memory</name>
http://git-wip-us.apache.org/repos/asf/arrow/blob/a8f8ba0c/java/pom.xml
----------------------------------------------------------------------
diff --git a/java/pom.xml b/java/pom.xml
index 1fa8ef9..8237a30 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0</version>
<packaging>pom</packaging>
<name>Apache Arrow Java Root POM</name>
@@ -42,7 +42,7 @@
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</connection>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</developerConnection>
<url>https://github.com/apache/arrow</url>
- <tag>HEAD</tag>
+ <tag>apache-arrow-0.4.0</tag>
</scm>
<mailingLists>
http://git-wip-us.apache.org/repos/asf/arrow/blob/a8f8ba0c/java/tools/pom.xml
----------------------------------------------------------------------
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index 6124b85..31f931b 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
<parent>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0</version>
</parent>
<artifactId>arrow-tools</artifactId>
<name>Arrow Tools</name>
http://git-wip-us.apache.org/repos/asf/arrow/blob/a8f8ba0c/java/vector/pom.xml
----------------------------------------------------------------------
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index e091936..2605523 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
<parent>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.4.0-SNAPSHOT</version>
+ <version>0.4.0</version>
</parent>
<artifactId>arrow-vector</artifactId>
<name>Arrow Vectors</name>
[12/14] arrow git commit: ARROW-1066: [Python] pandas 0.20.1
deprecation of pd.lib causes a warning on import
Posted by we...@apache.org.
ARROW-1066: [Python] pandas 0.20.1 deprecation of pd.lib causes a warning on import
Author: Jeff Reback <je...@twosigma.com>
Closes #715 from jreback/pandas_compat and squashes the following commits:
842b19f [Jeff Reback] COMPAT: pandas 0.20.1 deprecation of pd.lib causes a warning on import
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/078357af
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/078357af
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/078357af
Branch: refs/heads/master
Commit: 078357af32514502200a2fb83b0376c6c36f623c
Parents: 1cb18d5
Author: Jeff Reback <je...@twosigma.com>
Authored: Wed May 24 17:18:45 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Wed May 31 13:45:48 2017 -0400
----------------------------------------------------------------------
python/pyarrow/feather.py | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/078357af/python/pyarrow/feather.py
----------------------------------------------------------------------
diff --git a/python/pyarrow/feather.py b/python/pyarrow/feather.py
index 34783a7..9f40ab4 100644
--- a/python/pyarrow/feather.py
+++ b/python/pyarrow/feather.py
@@ -26,6 +26,11 @@ from pyarrow.lib import FeatherError # noqa
from pyarrow.lib import Table
import pyarrow.lib as ext
+try:
+ infer_dtype = pdapi.infer_dtype
+except AttributeError:
+ infer_dtype = pd.lib.infer_dtype
+
if LooseVersion(pd.__version__) < '0.17.0':
raise ImportError("feather requires pandas >= 0.17.0")
@@ -75,7 +80,7 @@ class FeatherWriter(object):
col = df.iloc[:, i]
if pdapi.is_object_dtype(col):
- inferred_type = pd.lib.infer_dtype(col)
+ inferred_type = infer_dtype(col)
msg = ("cannot serialize column {n} "
"named {name} with dtype {dtype}".format(
n=i, name=name, dtype=inferred_type))
@@ -83,7 +88,7 @@ class FeatherWriter(object):
if inferred_type in ['mixed']:
# allow columns with nulls + an inferable type
- inferred_type = pd.lib.infer_dtype(col[col.notnull()])
+ inferred_type = infer_dtype(col[col.notnull()])
if inferred_type in ['mixed']:
raise ValueError(msg)
[05/14] arrow git commit: ARROW-1061: [C++] Harden decimal parsing
against invalid strings
Posted by we...@apache.org.
ARROW-1061: [C++] Harden decimal parsing against invalid strings
Author: Phillip Cloud <cp...@gmail.com>
Closes #708 from cpcloud/ARROW-1061 and squashes the following commits:
c93d1db8 [Phillip Cloud] ARROW-1061: [C++] Harden decimal parsing against invalid strings
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/1cb18d52
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/1cb18d52
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/1cb18d52
Branch: refs/heads/master
Commit: 1cb18d528bbb61943273eb8ca8d9dd1a8c631485
Parents: 33117d9
Author: Phillip Cloud <cp...@gmail.com>
Authored: Tue May 23 20:29:44 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Wed May 31 13:45:48 2017 -0400
----------------------------------------------------------------------
cpp/src/arrow/util/decimal-test.cc | 80 +++++++++++++++++++++++++++++++++
cpp/src/arrow/util/decimal.cc | 75 ++++++++++++++++++++++++-------
2 files changed, 139 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/1cb18d52/cpp/src/arrow/util/decimal-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/decimal-test.cc b/cpp/src/arrow/util/decimal-test.cc
index 5d95c2c..72107a2 100644
--- a/cpp/src/arrow/util/decimal-test.cc
+++ b/cpp/src/arrow/util/decimal-test.cc
@@ -54,6 +54,29 @@ TYPED_TEST(DecimalTest, TestFromString) {
ASSERT_EQ(scale, 5);
}
+TEST(DecimalTest, TestStringStartingWithPlus) {
+ std::string plus_value("+234.234");
+ Decimal32 out;
+ int scale;
+ int precision;
+ ASSERT_OK(FromString(plus_value, &out, &precision, &scale));
+ ASSERT_EQ(234234, out.value);
+ ASSERT_EQ(6, precision);
+ ASSERT_EQ(3, scale);
+}
+
+TEST(DecimalTest, TestStringStartingWithPlus128) {
+ std::string plus_value("+2342394230592.232349023094");
+ decimal::int128_t expected_value("2342394230592232349023094");
+ Decimal128 out;
+ int scale;
+ int precision;
+ ASSERT_OK(FromString(plus_value, &out, &precision, &scale));
+ ASSERT_EQ(expected_value, out.value);
+ ASSERT_EQ(25, precision);
+ ASSERT_EQ(12, scale);
+}
+
TEST(DecimalTest, TestStringToInt32) {
int32_t value = 0;
StringToInteger("123", "456", 1, &value);
@@ -160,6 +183,63 @@ TEST(DecimalTest, TestDecimal128StringAndBytesRoundTrip) {
ASSERT_EQ(expected.value, result.value);
}
+TEST(DecimalTest, TestInvalidInputMinus) {
+ std::string invalid_value("-");
+ Decimal32 out;
+ Status status = decimal::FromString(invalid_value, &out);
+ ASSERT_RAISES(Invalid, status);
+}
+
+TEST(DecimalTest, TestInvalidInputDot) {
+ std::string invalid_value("0.0.0");
+ Decimal32 out;
+ Status status = decimal::FromString(invalid_value, &out);
+ ASSERT_RAISES(Invalid, status);
+}
+
+TEST(DecimalTest, TestInvalidInputEmbeddedMinus) {
+ std::string invalid_value("0-13-32");
+ Decimal32 out;
+ Status status = decimal::FromString(invalid_value, &out);
+ ASSERT_RAISES(Invalid, status);
+}
+
+TEST(DecimalTest, TestInvalidInputSingleChar) {
+ std::string invalid_value("a");
+ Decimal32 out;
+ Status status = decimal::FromString(invalid_value, &out);
+ ASSERT_RAISES(Invalid, status);
+}
+
+TEST(DecimalTest, TestInvalidInputWithValidSubstring) {
+ std::string invalid_value("-23092.235-");
+ Decimal32 out;
+ Status status = decimal::FromString(invalid_value, &out);
+ auto msg = status.message();
+ ASSERT_RAISES(Invalid, status);
+}
+
+TEST(DecimalTest, TestInvalidInputWithMinusPlus) {
+ std::string invalid_value("-+23092.235");
+ Decimal32 out;
+ Status status = decimal::FromString(invalid_value, &out);
+ ASSERT_RAISES(Invalid, status);
+}
+
+TEST(DecimalTest, TestInvalidInputWithPlusMinus) {
+ std::string invalid_value("+-23092.235");
+ Decimal32 out;
+ Status status = decimal::FromString(invalid_value, &out);
+ ASSERT_RAISES(Invalid, status);
+}
+
+TEST(DecimalTest, TestInvalidInputWithLeadingZeros) {
+ std::string invalid_value("00a");
+ Decimal32 out;
+ Status status = decimal::FromString(invalid_value, &out);
+ ASSERT_RAISES(Invalid, status);
+}
+
template <typename T>
class DecimalZerosTest : public ::testing::Test {};
TYPED_TEST_CASE(DecimalZerosTest, DecimalTypes);
http://git-wip-us.apache.org/repos/asf/arrow/blob/1cb18d52/cpp/src/arrow/util/decimal.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index 734df13..72ede35 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -29,18 +29,27 @@ ARROW_EXPORT Status FromString(
}
int8_t sign = 1;
- auto charp = s.c_str();
- auto end = charp + s.length();
+ std::string::const_iterator charp = s.cbegin();
+ std::string::const_iterator end = s.cend();
- if (*charp == '+' || *charp == '-') {
- if (*charp == '-') { sign = -1; }
+ char first_char = *charp;
+ if (first_char == '+' || first_char == '-') {
+ if (first_char == '-') { sign = -1; }
++charp;
}
- auto numeric_string_start = charp;
+ if (charp == end) {
+ std::stringstream ss;
+ ss << "Single character: '" << first_char << "' is not a valid decimal value";
+ return Status::Invalid(ss.str());
+ }
+
+ std::string::const_iterator numeric_string_start = charp;
+
+ DCHECK_LT(charp, end);
// skip leading zeros
- while (*charp == '0') {
+ while (charp != end && *charp == '0') {
++charp;
}
@@ -59,25 +68,59 @@ ARROW_EXPORT Status FromString(
return Status::OK();
}
- auto whole_part_start = charp;
- while (isdigit(*charp)) {
+ std::string::const_iterator whole_part_start = charp;
+
+ while (charp != end && isdigit(*charp)) {
++charp;
}
- auto whole_part_end = charp;
+
+ std::string::const_iterator whole_part_end = charp;
std::string whole_part(whole_part_start, whole_part_end);
- if (*charp == '.') {
+ if (charp != end && *charp == '.') {
++charp;
+
+ if (charp == end) {
+ return Status::Invalid(
+ "Decimal point must be followed by at least one base ten digit. Reached the "
+ "end of the string.");
+ }
+
+ if (!isdigit(*charp)) {
+ std::stringstream ss;
+ ss << "Decimal point must be followed by a base ten digit. Found '" << *charp
+ << "'";
+ return Status::Invalid(ss.str());
+ }
} else {
- // no decimal point
- DCHECK_EQ(charp, end);
+ if (charp != end) {
+ std::stringstream ss;
+ ss << "Expected base ten digit or decimal point but found '" << *charp
+ << "' instead.";
+ return Status::Invalid(ss.str());
+ }
}
- auto fractional_part_start = charp;
- while (isdigit(*charp)) {
- ++charp;
+ std::string::const_iterator fractional_part_start = charp;
+
+ // The rest must be digits, because if we have a decimal point it must be followed by
+ // digits
+ if (charp != end) {
+ while (charp != end && isdigit(*charp)) {
+ ++charp;
+ }
+
+ // The while loop has ended before the end of the string which means we've hit a
+ // character that isn't a base ten digit
+ if (charp != end) {
+ std::stringstream ss;
+ ss << "Found non base ten digit character '" << *charp
+ << "' before the end of the string";
+ return Status::Invalid(ss.str());
+ }
}
- auto fractional_part_end = charp;
+
+ std::string::const_iterator fractional_part_end = charp;
std::string fractional_part(fractional_part_start, fractional_part_end);
if (precision != nullptr) {
[06/14] arrow git commit: ARROW-1054: [Python] Fix test failure on
pandas 0.19.2, some refactoring
Posted by we...@apache.org.
ARROW-1054: [Python] Fix test failure on pandas 0.19.2, some refactoring
For esoteric reasons, `MultiIndex.from_arrays` rejects non-writeable NumPy arrays. This problem isn't present in pandas 0.20.1
Author: Wes McKinney <we...@twosigma.com>
Closes #705 from wesm/ARROW-1054 and squashes the following commits:
f7dfbb24 [Wes McKinney] Factor out table to blocks into pandas_compat, fix pandas 0.19.2 test failure
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/a6e77f4b
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/a6e77f4b
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/a6e77f4b
Branch: refs/heads/master
Commit: a6e77f4b2f4e3d5affc2aef3c5f18fa534b90472
Parents: cf4ef5e
Author: Wes McKinney <we...@twosigma.com>
Authored: Fri May 19 11:12:33 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Wed May 31 13:45:48 2017 -0400
----------------------------------------------------------------------
cpp/src/arrow/python/pandas_convert.cc | 2 +-
python/pyarrow/ipc.py | 3 +-
python/pyarrow/pandas_compat.py | 73 +++++++++++++++++++++++++++
python/pyarrow/parquet.py | 1 -
python/pyarrow/table.pxi | 76 ++---------------------------
5 files changed, 81 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/a6e77f4b/cpp/src/arrow/python/pandas_convert.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/python/pandas_convert.cc b/cpp/src/arrow/python/pandas_convert.cc
index 96dd09a..ac61cbc 100644
--- a/cpp/src/arrow/python/pandas_convert.cc
+++ b/cpp/src/arrow/python/pandas_convert.cc
@@ -1102,7 +1102,7 @@ static inline PyObject* NewArray1DFromType(
set_numpy_metadata(type, arrow_type, descr);
return PyArray_NewFromDescr(&PyArray_Type, descr, 1, dims, nullptr, data,
- NPY_ARRAY_OWNDATA | NPY_ARRAY_CARRAY, nullptr);
+ NPY_ARRAY_OWNDATA | NPY_ARRAY_CARRAY | NPY_ARRAY_WRITEABLE, nullptr);
}
class PandasBlock {
http://git-wip-us.apache.org/repos/asf/arrow/blob/a6e77f4b/python/pyarrow/ipc.py
----------------------------------------------------------------------
diff --git a/python/pyarrow/ipc.py b/python/pyarrow/ipc.py
index a61d746..6173299 100644
--- a/python/pyarrow/ipc.py
+++ b/python/pyarrow/ipc.py
@@ -158,4 +158,5 @@ def deserialize_pandas(buf, nthreads=1):
"""
buffer_reader = pa.BufferReader(buf)
reader = pa.RecordBatchFileReader(buffer_reader)
- return reader.read_all().to_pandas(nthreads=nthreads)
+ table = reader.read_all()
+ return table.to_pandas(nthreads=nthreads)
http://git-wip-us.apache.org/repos/asf/arrow/blob/a6e77f4b/python/pyarrow/pandas_compat.py
----------------------------------------------------------------------
diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py
index 255b31a..9711b72 100644
--- a/python/pyarrow/pandas_compat.py
+++ b/python/pyarrow/pandas_compat.py
@@ -102,3 +102,76 @@ def construct_metadata(df, index_levels, preserve_index):
}
).encode('utf8')
}
+
+
+def table_to_blockmanager(table, nthreads=1):
+ import pandas.core.internals as _int
+ from pyarrow.compat import DatetimeTZDtype
+ import pyarrow.lib as lib
+
+ block_table = table
+
+ index_columns = []
+ index_arrays = []
+ index_names = []
+ schema = table.schema
+ row_count = table.num_rows
+ metadata = schema.metadata
+
+ if metadata is not None and b'pandas' in metadata:
+ pandas_metadata = json.loads(metadata[b'pandas'].decode('utf8'))
+ index_columns = pandas_metadata['index_columns']
+
+ for name in index_columns:
+ i = schema.get_field_index(name)
+ if i != -1:
+ col = table.column(i)
+ index_name = (None if is_unnamed_index_level(name)
+ else name)
+ values = col.to_pandas().values
+ if not values.flags.writeable:
+ # ARROW-1054: in pandas 0.19.2, factorize will reject
+ # non-writeable arrays when calling MultiIndex.from_arrays
+ values = values.copy()
+
+ index_arrays.append(values)
+ index_names.append(index_name)
+ block_table = block_table.remove_column(
+ block_table.schema.get_field_index(name)
+ )
+
+ result = lib.table_to_blocks(block_table, nthreads)
+
+ blocks = []
+ for item in result:
+ block_arr = item['block']
+ placement = item['placement']
+ if 'dictionary' in item:
+ cat = pd.Categorical(block_arr,
+ categories=item['dictionary'],
+ ordered=False, fastpath=True)
+ block = _int.make_block(cat, placement=placement,
+ klass=_int.CategoricalBlock,
+ fastpath=True)
+ elif 'timezone' in item:
+ dtype = DatetimeTZDtype('ns', tz=item['timezone'])
+ block = _int.make_block(block_arr, placement=placement,
+ klass=_int.DatetimeTZBlock,
+ dtype=dtype, fastpath=True)
+ else:
+ block = _int.make_block(block_arr, placement=placement)
+ blocks.append(block)
+
+ if len(index_arrays) > 1:
+ index = pd.MultiIndex.from_arrays(index_arrays, names=index_names)
+ elif len(index_arrays) == 1:
+ index = pd.Index(index_arrays[0], name=index_names[0])
+ else:
+ index = pd.RangeIndex(row_count)
+
+ axes = [
+ [column.name for column in block_table.itercolumns()],
+ index
+ ]
+
+ return _int.BlockManager(blocks, axes)
http://git-wip-us.apache.org/repos/asf/arrow/blob/a6e77f4b/python/pyarrow/parquet.py
----------------------------------------------------------------------
diff --git a/python/pyarrow/parquet.py b/python/pyarrow/parquet.py
index f59a719..dc26dab 100644
--- a/python/pyarrow/parquet.py
+++ b/python/pyarrow/parquet.py
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-import itertools
import json
import six
http://git-wip-us.apache.org/repos/asf/arrow/blob/a6e77f4b/python/pyarrow/table.pxi
----------------------------------------------------------------------
diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi
index 25a4f84..3f67ba4 100644
--- a/python/pyarrow/table.pxi
+++ b/python/pyarrow/table.pxi
@@ -559,86 +559,20 @@ cdef class RecordBatch:
return pyarrow_wrap_batch(batch)
-cdef table_to_blockmanager(const shared_ptr[CTable]& ctable, int nthreads):
- import pandas.core.internals as _int
- from pandas import RangeIndex, Categorical
- from pyarrow.compat import DatetimeTZDtype
-
- cdef:
- Table table = pyarrow_wrap_table(ctable)
- Table block_table = pyarrow_wrap_table(ctable)
- Schema schema = table.schema
-
- size_t row_count = table.num_rows
- size_t total_columns = table.num_columns
-
- dict metadata = schema.metadata
- dict pandas_metadata = None
-
- list index_columns = []
- list index_arrays = []
-
- if metadata is not None and b'pandas' in metadata:
- pandas_metadata = json.loads(metadata[b'pandas'].decode('utf8'))
- index_columns = pandas_metadata['index_columns']
-
- cdef:
- Column col
- int64_t i
-
- for name in index_columns:
- i = schema.get_field_index(name)
- if i != -1:
- col = table.column(i)
- index_name = None if pdcompat.is_unnamed_index_level(name) else name
- index_arrays.append(
- pd.Index(col.to_pandas().values, name=index_name)
- )
- block_table = block_table.remove_column(
- block_table.schema.get_field_index(name)
- )
-
+def table_to_blocks(Table table, int nthreads):
cdef:
PyObject* result_obj
- shared_ptr[CTable] c_block_table = block_table.sp_table
+ shared_ptr[CTable] c_table = table.sp_table
with nogil:
check_status(
libarrow.ConvertTableToPandas(
- c_block_table, nthreads, &result_obj
+ c_table, nthreads, &result_obj
)
)
- result = PyObject_to_object(result_obj)
-
- blocks = []
- for item in result:
- block_arr = item['block']
- placement = item['placement']
- if 'dictionary' in item:
- cat = Categorical(block_arr,
- categories=item['dictionary'],
- ordered=False, fastpath=True)
- block = _int.make_block(cat, placement=placement,
- klass=_int.CategoricalBlock,
- fastpath=True)
- elif 'timezone' in item:
- dtype = DatetimeTZDtype('ns', tz=item['timezone'])
- block = _int.make_block(block_arr, placement=placement,
- klass=_int.DatetimeTZBlock,
- dtype=dtype, fastpath=True)
- else:
- block = _int.make_block(block_arr, placement=placement)
- blocks.append(block)
-
- cdef list axes = [
- [column.name for column in block_table.itercolumns()],
- pd.MultiIndex.from_arrays(
- index_arrays
- ) if index_arrays else pd.RangeIndex(row_count),
- ]
+ return PyObject_to_object(result_obj)
- return _int.BlockManager(blocks, axes)
cdef class Table:
@@ -829,7 +763,7 @@ cdef class Table:
if nthreads is None:
nthreads = cpu_count()
- mgr = table_to_blockmanager(self.sp_table, nthreads)
+ mgr = pdcompat.table_to_blockmanager(self, nthreads)
return pd.DataFrame(mgr)
def to_pydict(self):
[08/14] arrow git commit: ARROW-1062: [GLib] Follow API changes in
examples
Posted by we...@apache.org.
ARROW-1062: [GLib] Follow API changes in examples
Author: Kouhei Sutou <ko...@clear-code.com>
Closes #709 from kou/glib-fix-examples and squashes the following commits:
ed5c465d [Kouhei Sutou] [GLib] Follow API changes in examples
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/d2cc199c
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/d2cc199c
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/d2cc199c
Branch: refs/heads/master
Commit: d2cc199cc480f206110b34dc6abb0e3eb51eacac
Parents: b06602d
Author: Kouhei Sutou <ko...@clear-code.com>
Authored: Tue May 23 14:05:27 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Wed May 31 13:45:48 2017 -0400
----------------------------------------------------------------------
.travis.yml | 1 +
c_glib/arrow-glib/table.h | 2 --
c_glib/example/go/read-batch.go | 2 +-
c_glib/example/go/read-stream.go | 2 +-
c_glib/example/go/write-batch.go | 2 +-
c_glib/example/go/write-stream.go | 2 +-
c_glib/example/lua/read-batch.lua | 2 +-
c_glib/example/lua/read-stream.lua | 2 +-
c_glib/example/lua/stream-to-torch-tensor.lua | 2 +-
c_glib/example/lua/write-batch.lua | 2 +-
c_glib/example/lua/write-stream.lua | 2 +-
ci/travis_before_script_c_glib.sh | 24 ++++++++++++++++++---
ci/travis_env_common.sh | 1 +
ci/travis_script_c_glib.sh | 25 +++++++++++++++++++++-
14 files changed, 56 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/d2cc199c/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 022ffb3..b212fcb 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,6 +17,7 @@ addons:
- gtk-doc-tools
- autoconf-archive
- libgirepository1.0-dev
+ - gir1.2-gudev-1.0
services:
- docker
http://git-wip-us.apache.org/repos/asf/arrow/blob/d2cc199c/c_glib/arrow-glib/table.h
----------------------------------------------------------------------
diff --git a/c_glib/arrow-glib/table.h b/c_glib/arrow-glib/table.h
index 7f83872..9e21669 100644
--- a/c_glib/arrow-glib/table.h
+++ b/c_glib/arrow-glib/table.h
@@ -86,6 +86,4 @@ GArrowTable *garrow_table_remove_column (GArrowTable *table,
guint i,
GError **error);
-gchar *garrow_table_to_string (GArrowTable *table);
-
G_END_DECLS
http://git-wip-us.apache.org/repos/asf/arrow/blob/d2cc199c/c_glib/example/go/read-batch.go
----------------------------------------------------------------------
diff --git a/c_glib/example/go/read-batch.go b/c_glib/example/go/read-batch.go
index 3ad8423..f47d3e2 100644
--- a/c_glib/example/go/read-batch.go
+++ b/c_glib/example/go/read-batch.go
@@ -82,7 +82,7 @@ func main() {
if err != nil {
log.Fatalf("Failed to open path: <%s>: %v", path, err)
}
- reader, err := arrow.NewFileReader(input)
+ reader, err := arrow.NewRecordBatchFileReader(input)
if err != nil {
log.Fatalf("Failed to parse data: %v", err)
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/d2cc199c/c_glib/example/go/read-stream.go
----------------------------------------------------------------------
diff --git a/c_glib/example/go/read-stream.go b/c_glib/example/go/read-stream.go
index f336798..789b87b 100644
--- a/c_glib/example/go/read-stream.go
+++ b/c_glib/example/go/read-stream.go
@@ -82,7 +82,7 @@ func main() {
if err != nil {
log.Fatalf("Failed to open path: <%s>: %v", path, err)
}
- reader, err := arrow.NewStreamReader(input)
+ reader, err := arrow.NewRecordBatchStreamReader(input)
if err != nil {
log.Fatalf("Failed to parse data: %v", err)
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/d2cc199c/c_glib/example/go/write-batch.go
----------------------------------------------------------------------
diff --git a/c_glib/example/go/write-batch.go b/c_glib/example/go/write-batch.go
index 78ef1be..cda09a9 100644
--- a/c_glib/example/go/write-batch.go
+++ b/c_glib/example/go/write-batch.go
@@ -130,7 +130,7 @@ func main() {
if err != nil {
log.Fatalf("Failed to open path: <%s>: %v", output_path, err)
}
- writer, err := arrow.NewFileWriter(output, schema)
+ writer, err := arrow.NewRecordBatchFileWriter(output, schema)
if err != nil {
log.Fatalf("Failed to create writer: %v", err)
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/d2cc199c/c_glib/example/go/write-stream.go
----------------------------------------------------------------------
diff --git a/c_glib/example/go/write-stream.go b/c_glib/example/go/write-stream.go
index 3f08270..20cb03e 100644
--- a/c_glib/example/go/write-stream.go
+++ b/c_glib/example/go/write-stream.go
@@ -130,7 +130,7 @@ func main() {
if err != nil {
log.Fatalf("Failed to open path: <%s>: %v", output_path, err)
}
- writer, err := arrow.NewStreamWriter(output, schema)
+ writer, err := arrow.NewRecordBatchStreamWriter(output, schema)
if err != nil {
log.Fatalf("Failed to create writer: %v", err)
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/d2cc199c/c_glib/example/lua/read-batch.lua
----------------------------------------------------------------------
diff --git a/c_glib/example/lua/read-batch.lua b/c_glib/example/lua/read-batch.lua
index 090a857..5e28d7e 100644
--- a/c_glib/example/lua/read-batch.lua
+++ b/c_glib/example/lua/read-batch.lua
@@ -21,7 +21,7 @@ local Arrow = lgi.Arrow
local input_path = arg[1] or "/tmp/batch.arrow";
local input = Arrow.MemoryMappedInputStream.new(input_path)
-local reader = Arrow.FileReader.new(input)
+local reader = Arrow.RecordBatchFileReader.new(input)
for i = 0, reader:get_n_record_batches() - 1 do
local record_batch = reader:get_record_batch(i)
http://git-wip-us.apache.org/repos/asf/arrow/blob/d2cc199c/c_glib/example/lua/read-stream.lua
----------------------------------------------------------------------
diff --git a/c_glib/example/lua/read-stream.lua b/c_glib/example/lua/read-stream.lua
index d7ac5eb..95cf2e7 100644
--- a/c_glib/example/lua/read-stream.lua
+++ b/c_glib/example/lua/read-stream.lua
@@ -21,7 +21,7 @@ local Arrow = lgi.Arrow
local input_path = arg[1] or "/tmp/stream.arrow";
local input = Arrow.MemoryMappedInputStream.new(input_path)
-local reader = Arrow.StreamReader.new(input)
+local reader = Arrow.RecordBatchStreamReader.new(input)
local i = 0
while true do
http://git-wip-us.apache.org/repos/asf/arrow/blob/d2cc199c/c_glib/example/lua/stream-to-torch-tensor.lua
----------------------------------------------------------------------
diff --git a/c_glib/example/lua/stream-to-torch-tensor.lua b/c_glib/example/lua/stream-to-torch-tensor.lua
index 237d759..9825f10 100644
--- a/c_glib/example/lua/stream-to-torch-tensor.lua
+++ b/c_glib/example/lua/stream-to-torch-tensor.lua
@@ -77,7 +77,7 @@ end
local input_path = arg[1] or "/tmp/stream.arrow";
local input = Arrow.MemoryMappedInputStream.new(input_path)
-local reader = Arrow.StreamReader.open(input)
+local reader = Arrow.RecordBatchStreamReader.new(input)
local i = 0
while true do
http://git-wip-us.apache.org/repos/asf/arrow/blob/d2cc199c/c_glib/example/lua/write-batch.lua
----------------------------------------------------------------------
diff --git a/c_glib/example/lua/write-batch.lua b/c_glib/example/lua/write-batch.lua
index 663f8ef..26acc1d 100644
--- a/c_glib/example/lua/write-batch.lua
+++ b/c_glib/example/lua/write-batch.lua
@@ -35,7 +35,7 @@ local fields = {
local schema = Arrow.Schema.new(fields)
local output = Arrow.FileOutputStream.new(output_path, false)
-local writer = Arrow.FileWriter.new(output, schema)
+local writer = Arrow.RecordBatchFileWriter.new(output, schema)
function build_array(builder, values)
for _, value in pairs(values) do
http://git-wip-us.apache.org/repos/asf/arrow/blob/d2cc199c/c_glib/example/lua/write-stream.lua
----------------------------------------------------------------------
diff --git a/c_glib/example/lua/write-stream.lua b/c_glib/example/lua/write-stream.lua
index fb6cc55..07bbd79 100644
--- a/c_glib/example/lua/write-stream.lua
+++ b/c_glib/example/lua/write-stream.lua
@@ -35,7 +35,7 @@ local fields = {
local schema = Arrow.Schema.new(fields)
local output = Arrow.FileOutputStream.new(output_path, false)
-local writer = Arrow.StreamWriter.new(output, schema)
+local writer = Arrow.RecordBatchStreamWriter.new(output, schema)
function build_array(builder, values)
for _, value in pairs(values) do
http://git-wip-us.apache.org/repos/asf/arrow/blob/d2cc199c/ci/travis_before_script_c_glib.sh
----------------------------------------------------------------------
diff --git a/ci/travis_before_script_c_glib.sh b/ci/travis_before_script_c_glib.sh
index 74bdd94..e397b0e 100755
--- a/ci/travis_before_script_c_glib.sh
+++ b/ci/travis_before_script_c_glib.sh
@@ -18,14 +18,32 @@ set -ex
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
if [ $TRAVIS_OS_NAME == "osx" ]; then
- brew install gtk-doc autoconf-archive gobject-introspection
+ brew install gtk-doc autoconf-archive gobject-introspection
fi
gem install gobject-introspection
-pushd $ARROW_C_GLIB_DIR
+git clone \
+ --quiet \
+ --depth 1 \
+ --recursive \
+ https://github.com/torch/distro.git ~/torch
+pushd ~/torch
+./install-deps > /dev/null
+echo "yes" | ./install.sh > /dev/null
+. ~/torch/install/bin/torch-activate
+popd
+luarocks install lgi
+
+go get github.com/linuxdeepin/go-gir-generator || :
+pushd $GOPATH/src/github.com/linuxdeepin/go-gir-generator
+make build copyfile
+mkdir -p $GOPATH/bin/
+cp -a out/gir-generator $GOPATH/bin/
+cp -a out/src/gir/ $GOPATH/src/
+popd
-: ${ARROW_C_GLIB_INSTALL=$TRAVIS_BUILD_DIR/c-glib-install}
+pushd $ARROW_C_GLIB_DIR
./autogen.sh
http://git-wip-us.apache.org/repos/asf/arrow/blob/d2cc199c/ci/travis_env_common.sh
----------------------------------------------------------------------
diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh
index 3515314..b19fafa 100755
--- a/ci/travis_env_common.sh
+++ b/ci/travis_env_common.sh
@@ -30,3 +30,4 @@ export ARROW_INTEGRATION_DIR=$TRAVIS_BUILD_DIR/integration
export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install
export ARROW_CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build
+export ARROW_C_GLIB_INSTALL=$TRAVIS_BUILD_DIR/c-glib-install
http://git-wip-us.apache.org/repos/asf/arrow/blob/d2cc199c/ci/travis_script_c_glib.sh
----------------------------------------------------------------------
diff --git a/ci/travis_script_c_glib.sh b/ci/travis_script_c_glib.sh
index 1492354..c4df713 100755
--- a/ci/travis_script_c_glib.sh
+++ b/ci/travis_script_c_glib.sh
@@ -14,11 +14,34 @@
set -e
-ARROW_C_GLIB_DIR=$TRAVIS_BUILD_DIR/c_glib
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
pushd $ARROW_C_GLIB_DIR
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_CPP_INSTALL/lib
NO_MAKE=yes test/run-test.sh
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ARROW_C_GLIB_INSTALL/lib
+export GI_TYPELIB_PATH=$ARROW_C_GLIB_INSTALL/lib/girepository-1.0
+export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ARROW_CPP_INSTALL/lib/pkgconfig
+export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$ARROW_C_GLIB_INSTALL/lib/pkgconfig
+
+pushd example/lua
+. ~/torch/install/bin/torch-activate
+luajit write-batch.lua
+luajit read-batch.lua
+luajit write-stream.lua
+luajit read-stream.lua
+luajit stream-to-torch-tensor.lua
+popd
+
+pushd example/go
+make generate
+make
+./write-batch
+./read-batch
+./write-stream
+./read-stream
+popd
+
popd
[11/14] arrow git commit: ARROW-1060: [Python] Add unit tests for
reference counts in memoryview interface
Posted by we...@apache.org.
ARROW-1060: [Python] Add unit tests for reference counts in memoryview interface
Add missing test case for ARROW-1053.
Author: Wes McKinney <we...@twosigma.com>
Closes #710 from wesm/ARROW-1060 and squashes the following commits:
013fc248 [Wes McKinney] Add unit tests to check that reference counts stay constant via memoryview interface
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/aa652cb3
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/aa652cb3
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/aa652cb3
Branch: refs/heads/master
Commit: aa652cb3f5ea4b1f845ea2ef3e10344b050f45b3
Parents: 84b7ee1
Author: Wes McKinney <we...@twosigma.com>
Authored: Tue May 23 14:08:39 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Wed May 31 13:45:48 2017 -0400
----------------------------------------------------------------------
python/pyarrow/tests/test_io.py | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/aa652cb3/python/pyarrow/tests/test_io.py
----------------------------------------------------------------------
diff --git a/python/pyarrow/tests/test_io.py b/python/pyarrow/tests/test_io.py
index f91dc6f..397b7a8 100644
--- a/python/pyarrow/tests/test_io.py
+++ b/python/pyarrow/tests/test_io.py
@@ -19,6 +19,7 @@ from io import BytesIO
import gc
import os
import pytest
+import sys
import numpy as np
@@ -201,8 +202,6 @@ def test_inmemory_write_after_closed():
def test_buffer_protocol_ref_counting():
- import gc
-
def make_buffer(bytes_obj):
return bytearray(pa.frombuffer(bytes_obj))
@@ -210,6 +209,14 @@ def test_buffer_protocol_ref_counting():
gc.collect()
assert buf == b'foo'
+ # ARROW-1053
+ val = b'foo'
+ refcount_before = sys.getrefcount(val)
+ for i in range(10):
+ make_buffer(val)
+ gc.collect()
+ assert refcount_before == sys.getrefcount(val)
+
def test_nativefile_write_memoryview():
f = pa.BufferOutputStream()
[14/14] arrow git commit: ARROW-424: [C++] Make ReadAt,
Write HDFS functions threadsafe
Posted by we...@apache.org.
ARROW-424: [C++] Make ReadAt, Write HDFS functions threadsafe
This also fixes the HDFS test suite to actually use libhdfs3 (it was not by accident)
Author: Wes McKinney <we...@twosigma.com>
Closes #712 from wesm/ARROW-424 and squashes the following commits:
0894719 [Wes McKinney] Make ReadAt, Write HDFS functions threadsafe
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/4e4435ec
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/4e4435ec
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/4e4435ec
Branch: refs/heads/master
Commit: 4e4435ecc4e4a02a9029e65c312bdfb00af7e291
Parents: 078357a
Author: Wes McKinney <we...@twosigma.com>
Authored: Wed May 24 17:20:40 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Wed May 31 13:45:48 2017 -0400
----------------------------------------------------------------------
cpp/src/arrow/array.h | 1 -
cpp/src/arrow/io/hdfs.cc | 5 +++
cpp/src/arrow/io/io-hdfs-test.cc | 68 ++++++++++++++++++++++++++++++-----
3 files changed, 64 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/4e4435ec/cpp/src/arrow/array.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
index 331c6c3..2c96ce0 100644
--- a/cpp/src/arrow/array.h
+++ b/cpp/src/arrow/array.h
@@ -508,7 +508,6 @@ ARROW_EXTERN_TEMPLATE NumericArray<Time32Type>;
ARROW_EXTERN_TEMPLATE NumericArray<Time64Type>;
ARROW_EXTERN_TEMPLATE NumericArray<TimestampType>;
-
/// \brief Perform any validation checks to determine obvious inconsistencies
/// with the array's internal data
///
http://git-wip-us.apache.org/repos/asf/arrow/blob/4e4435ec/cpp/src/arrow/io/hdfs.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index a27e132..ba9c2c2 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -89,6 +89,9 @@ class HdfsAnyFileImpl {
LibHdfsShim* driver_;
+ // For threadsafety
+ std::mutex lock_;
+
// These are pointers in libhdfs, so OK to copy
hdfsFS fs_;
hdfsFile file_;
@@ -116,6 +119,7 @@ class HdfsReadableFile::HdfsReadableFileImpl : public HdfsAnyFileImpl {
ret = driver_->Pread(fs_, file_, static_cast<tOffset>(position),
reinterpret_cast<void*>(buffer), static_cast<tSize>(nbytes));
} else {
+ std::lock_guard<std::mutex> guard(lock_);
RETURN_NOT_OK(Seek(position));
return Read(nbytes, bytes_read, buffer);
}
@@ -253,6 +257,7 @@ class HdfsOutputStream::HdfsOutputStreamImpl : public HdfsAnyFileImpl {
}
Status Write(const uint8_t* buffer, int64_t nbytes, int64_t* bytes_written) {
+ std::lock_guard<std::mutex> guard(lock_);
tSize ret = driver_->Write(
fs_, file_, reinterpret_cast<const void*>(buffer), static_cast<tSize>(nbytes));
CHECK_FAILURE(ret, "Write");
http://git-wip-us.apache.org/repos/asf/arrow/blob/4e4435ec/cpp/src/arrow/io/io-hdfs-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/io/io-hdfs-test.cc b/cpp/src/arrow/io/io-hdfs-test.cc
index 0fdb897..74f8042 100644
--- a/cpp/src/arrow/io/io-hdfs-test.cc
+++ b/cpp/src/arrow/io/io-hdfs-test.cc
@@ -19,6 +19,7 @@
#include <iostream>
#include <sstream>
#include <string>
+#include <thread>
#include "gtest/gtest.h"
@@ -38,6 +39,14 @@ std::vector<uint8_t> RandomData(int64_t size) {
return buffer;
}
+struct JNIDriver {
+ static HdfsDriver type;
+};
+
+struct PivotalDriver {
+ static HdfsDriver type;
+};
+
template <typename DRIVER>
class TestHdfsClient : public ::testing::Test {
public:
@@ -112,6 +121,7 @@ class TestHdfsClient : public ::testing::Test {
conf_.host = host == nullptr ? "localhost" : host;
conf_.user = user;
conf_.port = port == nullptr ? 20500 : atoi(port);
+ conf_.driver = DRIVER::type;
ASSERT_OK(HdfsClient::Connect(&conf_, &client_));
}
@@ -133,20 +143,19 @@ class TestHdfsClient : public ::testing::Test {
std::shared_ptr<HdfsClient> client_;
};
+template <>
+std::string TestHdfsClient<PivotalDriver>::HdfsAbsPath(const std::string& relpath) {
+ std::stringstream ss;
+ ss << relpath;
+ return ss.str();
+}
+
#define SKIP_IF_NO_DRIVER() \
if (!this->loaded_driver_) { \
std::cout << "Driver not loaded, skipping" << std::endl; \
return; \
}
-struct JNIDriver {
- static HdfsDriver type;
-};
-
-struct PivotalDriver {
- static HdfsDriver type;
-};
-
HdfsDriver JNIDriver::type = HdfsDriver::LIBHDFS;
HdfsDriver PivotalDriver::type = HdfsDriver::LIBHDFS3;
@@ -364,7 +373,6 @@ TYPED_TEST(TestHdfsClient, LargeFile) {
TYPED_TEST(TestHdfsClient, RenameFile) {
SKIP_IF_NO_DRIVER();
-
ASSERT_OK(this->MakeScratchDir());
auto src_path = this->ScratchPath("src-file");
@@ -380,5 +388,47 @@ TYPED_TEST(TestHdfsClient, RenameFile) {
ASSERT_TRUE(this->client_->Exists(dst_path));
}
+TYPED_TEST(TestHdfsClient, ThreadSafety) {
+ SKIP_IF_NO_DRIVER();
+ ASSERT_OK(this->MakeScratchDir());
+
+ auto src_path = this->ScratchPath("threadsafety");
+
+ std::string data = "foobar";
+ ASSERT_OK(this->WriteDummyFile(src_path, reinterpret_cast<const uint8_t*>(data.c_str()),
+ static_cast<int64_t>(data.size())));
+
+ std::shared_ptr<HdfsReadableFile> file;
+ ASSERT_OK(this->client_->OpenReadable(src_path, &file));
+
+ std::atomic<int> correct_count(0);
+ const int niter = 1000;
+
+ auto ReadData = [&file, &correct_count, &data, niter]() {
+ for (int i = 0; i < niter; ++i) {
+ std::shared_ptr<Buffer> buffer;
+ if (i % 2 == 0) {
+ ASSERT_OK(file->ReadAt(3, 3, &buffer));
+ if (0 == memcmp(data.c_str() + 3, buffer->data(), 3)) { correct_count += 1; }
+ } else {
+ ASSERT_OK(file->ReadAt(0, 4, &buffer));
+ if (0 == memcmp(data.c_str() + 0, buffer->data(), 4)) { correct_count += 1; }
+ }
+ }
+ };
+
+ std::thread thread1(ReadData);
+ std::thread thread2(ReadData);
+ std::thread thread3(ReadData);
+ std::thread thread4(ReadData);
+
+ thread1.join();
+ thread2.join();
+ thread3.join();
+ thread4.join();
+
+ ASSERT_EQ(niter * 4, correct_count);
+}
+
} // namespace io
} // namespace arrow
[10/14] arrow git commit: ARROW-1063: [Website] Updates for 0.4.0
release, release posting
Posted by we...@apache.org.
ARROW-1063: [Website] Updates for 0.4.0 release, release posting
Author: Wes McKinney <we...@twosigma.com>
Closes #711 from wesm/ARROW-1063 and squashes the following commits:
90f33ac [Wes McKinney] Update to use dynamic mirror links
62a04e8 [Wes McKinney] Use Apache mirrors for download links
ae88978 [Wes McKinney] Finish blog draft, website updates
c7adc0f [Wes McKinney] Start 0.4.0 items
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/8a700ccd
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/8a700ccd
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/8a700ccd
Branch: refs/heads/master
Commit: 8a700ccdad745c250fe5d91a9104e7c2d6364c1b
Parents: 4e4435e
Author: Wes McKinney <we...@twosigma.com>
Authored: Wed May 24 17:25:42 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Wed May 31 13:45:48 2017 -0400
----------------------------------------------------------------------
site/_config.yml | 4 +
site/_includes/top.html | 1 +
site/_posts/2017-05-23-0.4.0-release.md | 111 ++++++++++++++
site/_release/0.2.0.md | 8 +-
site/_release/0.3.0.md | 10 +-
site/_release/0.4.0.md | 138 ++++++++++++++++++
site/_release/index.md | 4 +-
site/css/syntax.css | 209 +++++++++++++++++++++++++++
site/index.html | 10 +-
site/install.md | 27 ++--
10 files changed, 495 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/8a700ccd/site/_config.yml
----------------------------------------------------------------------
diff --git a/site/_config.yml b/site/_config.yml
index 8bb969a..fcb76a3 100644
--- a/site/_config.yml
+++ b/site/_config.yml
@@ -19,6 +19,10 @@ repository: https://github.com/apache/arrow
destination: build
excerpt_separator: ""
+kramdown:
+ input: GFM
+ syntax_highlighter: rouge
+
exclude:
- Gemfile
- Gemfile.lock
http://git-wip-us.apache.org/repos/asf/arrow/blob/8a700ccd/site/_includes/top.html
----------------------------------------------------------------------
diff --git a/site/_includes/top.html b/site/_includes/top.html
index cc537ba..cfc4cde 100644
--- a/site/_includes/top.html
+++ b/site/_includes/top.html
@@ -13,6 +13,7 @@
<link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
<link href="{{ site.baseurl }}/css/main.css" rel="stylesheet">
+ <link href="{{ site.baseurl }}/css/syntax.css" rel="stylesheet">
<script src="https://code.jquery.com/jquery-3.2.1.min.js"
integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4="
crossorigin="anonymous"></script>
http://git-wip-us.apache.org/repos/asf/arrow/blob/8a700ccd/site/_posts/2017-05-23-0.4.0-release.md
----------------------------------------------------------------------
diff --git a/site/_posts/2017-05-23-0.4.0-release.md b/site/_posts/2017-05-23-0.4.0-release.md
new file mode 100644
index 0000000..d94a68a
--- /dev/null
+++ b/site/_posts/2017-05-23-0.4.0-release.md
@@ -0,0 +1,111 @@
+---
+layout: post
+title: "Apache Arrow 0.4.0 Release"
+date: "2017-05-23 00:00:00 -0400"
+author: wesm
+categories: [release]
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+The Apache Arrow team is pleased to announce the 0.4.0 release of the
+project. While only 17 days since the release, it includes [**77 resolved
+JIRAs**][1] with some important new features and bug fixes.
+
+See the [Install Page][6] to learn how to get the libraries for your platform.
+
+### Expanded JavaScript Implementation
+
+The TypeScript Arrow implementation has undergone some work since 0.3.0 and can
+now read a substantial portion of the Arrow streaming binary format. As this
+implementation develops, we will eventually want to include JS in the
+integration test suite along with Java and C++ to ensure wire
+cross-compatibility.
+
+### Python Support for Apache Parquet on Windows
+
+With the [1.1.0 C++ release][7] of [Apache Parquet][8], we have enabled the
+`pyarrow.parquet` extension on Windows for Python 3.5 and 3.6. This should
+appear in conda-forge packages and PyPI in the near future. Developers can
+follow the [source build instructions][9].
+
+### Generalizing Arrow Streams
+
+In the 0.2.0 release, we defined the first version of the Arrow streaming
+binary format for low-cost messaging with columnar data. These streams presume
+that the message components are written as a continuous byte stream over a
+socket or file.
+
+We would like to be able to support other other transport protocols, like
+[gRPC][3], for the message components of Arrow streams. To that end, in C++ we
+defined an abstract stream reader interface, for which the current contiguous
+streaming format is one implementation:
+
+{% highlight cpp %}
+class RecordBatchReader {
+ public:
+ virtual std::shared_ptr<Schema> schema() const = 0;
+ virtual Status GetNextRecordBatch(std::shared_ptr<RecordBatch>* batch) = 0;
+};
+{% endhighlight %}
+
+It would also be good to define abstract stream reader and writer interfaces in
+the Java implementation.
+
+In an upcoming blog post, we will explain in more depth how Arrow streams work,
+but you can learn more about them by reading the [IPC specification][4].
+
+### C++ and Cython API for Python Extensions
+
+As other Python libraries with C or C++ extensions use Apache Arrow, they will
+need to be able to return Python objects wrapping the underlying C++
+objects. In this release, we have implemented a prototype C++ API which enables
+Python wrapper objects to be constructed from C++ extension code:
+
+{% highlight cpp %}
+#include "arrow/python/pyarrow.h"
+
+if (!arrow::py::import_pyarrow()) {
+ // Error
+}
+
+std::shared_ptr<arrow::RecordBatch> cpp_batch = GetData(...);
+PyObject* py_batch = arrow::py::wrap_batch(cpp_batch);
+{% endhighlight %}
+
+This API is intended to be usable from Cython code as well:
+
+{% highlight cython %}
+cimport pyarrow
+pyarrow.import_pyarrow()
+{% endhighlight %}
+
+### Python Wheel Installers on macOS
+
+With this release, `pip install pyarrow` works on macOS (OS X) as well as
+Linux. We are working on providing binary wheel installers for Windows as well.
+
+[1]: https://issues.apache.org/jira/issues/?jql=project%20%3D%20ARROW%20AND%20status%20in%20(Resolved%2C%20Closed)%20AND%20fixVersion%20%3D%200.4.0
+[2]: http://arrow.apache.org/install
+[3]: http://grpc.io/
+[4]: http://arrow.apache.org/docs/ipc.html
+[6]: http://arrow.apache.org/install
+[7]: https://github.com/apache/parquet-cpp/releases/tag/apache-parquet-cpp-1.1.0
+[8]: http://parquet.apache.org
+[9]: http://arrow.apache.org/docs/python/development.html
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/arrow/blob/8a700ccd/site/_release/0.2.0.md
----------------------------------------------------------------------
diff --git a/site/_release/0.2.0.md b/site/_release/0.2.0.md
index ddac64e..72bce7d 100644
--- a/site/_release/0.2.0.md
+++ b/site/_release/0.2.0.md
@@ -26,8 +26,8 @@ limitations under the License.
## Download
-* **Source Release**: [apache-arrow-0.2.0.tar.gz][6]
-* **Verification**: [md5][3], [asc][7]
+* [**Source Artifacts**][6]
+* [Git tag][2]
# Changelog
@@ -252,6 +252,4 @@ $ git shortlog -sn apache-arrow-0.1.0..apache-arrow-0.2.0
* [ARROW-561](https://issues.apache.org/jira/browse/ARROW-561) - Update java & python dependencies to improve downstream packaging experience
[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.2.0
-[3]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.2.0/apache-arrow-0.2.0.tar.gz.md5
-[6]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.2.0/apache-arrow-0.2.0.tar.gz
-[7]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.2.0/apache-arrow-0.2.0.tar.gz.asc
+[6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.2.0/
http://git-wip-us.apache.org/repos/asf/arrow/blob/8a700ccd/site/_release/0.3.0.md
----------------------------------------------------------------------
diff --git a/site/_release/0.3.0.md b/site/_release/0.3.0.md
index 443bcad..a88ce4d 100644
--- a/site/_release/0.3.0.md
+++ b/site/_release/0.3.0.md
@@ -24,10 +24,11 @@ limitations under the License.
# Apache Arrow 0.3.0 (5 May 2017)
+Read more in the [release blog post][8]
+
## Download
-* **Source Release**: [apache-arrow-0.3.0.tar.gz][6]
-* **Verification**: [md5][3], [asc][7]
+* [**Source Artifacts**][6]
* [Git tag d8db8f8][2]
# Changelog
@@ -360,6 +361,5 @@ $ git shortlog -sn apache-arrow-0.2.0..apache-arrow-0.3.0
* [ARROW-938](https://issues.apache.org/jira/browse/ARROW-938) - Fix Apache Rat errors from source release build
[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.3.0
-[3]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz.md5
-[6]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz
-[7]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz.asc
+[6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.3.0/
+[8]: http://arrow.apache.org/blog/2017/05/08/0.3-release/
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/arrow/blob/8a700ccd/site/_release/0.4.0.md
----------------------------------------------------------------------
diff --git a/site/_release/0.4.0.md b/site/_release/0.4.0.md
new file mode 100644
index 0000000..d0060a3
--- /dev/null
+++ b/site/_release/0.4.0.md
@@ -0,0 +1,138 @@
+---
+layout: default
+title: 0.4.0 Release
+permalink: /release/0.4.0.html
+---
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+# Apache Arrow 0.4.0 (22 May 2017)
+
+Read more in the [release blog post][8]
+
+## Download
+
+* [**Source Artifacts**][6]
+* [Git tag a8f8ba0][2]
+
+# Changelog
+
+## Contributors
+
+```shell
+$ git shortlog -sn apache-arrow-0.3.0..apache-arrow-0.4.0
+ 28 Wes McKinney
+ 18 Kouhei Sutou
+ 9 Uwe L. Korn
+ 3 Brian Hulette
+ 3 Emilio Lahr-Vivaz
+ 3 Philipp Moritz
+ 3 Phillip Cloud
+ 2 Julien Le Dem
+ 1 Bryan Cutler
+ 1 Jeff Reback
+ 1 Max Risuhin
+ 1 Wenchen Fan
+ 1 bgosztonyi
+```
+## New Features and Improvements
+
+* [ARROW-1000](https://issues.apache.org/jira/browse/ARROW-1000) - [GLib] Move install document to Website
+* [ARROW-1001](https://issues.apache.org/jira/browse/ARROW-1001) - [GLib] Unify writer files
+* [ARROW-1002](https://issues.apache.org/jira/browse/ARROW-1002) - [C++] It is not necessary to add padding after the magic header in the FileWriter implementation
+* [ARROW-1008](https://issues.apache.org/jira/browse/ARROW-1008) - [C++] Define abstract interface for stream iteration
+* [ARROW-1010](https://issues.apache.org/jira/browse/ARROW-1010) - [Website] Only show English posts in /blog/
+* [ARROW-1011](https://issues.apache.org/jira/browse/ARROW-1011) - [Format] Clarify requirements around buffer padding in validity bitmaps
+* [ARROW-1015](https://issues.apache.org/jira/browse/ARROW-1015) - [Java] Implement schema-level metadata
+* [ARROW-1016](https://issues.apache.org/jira/browse/ARROW-1016) - Python: Include C++ headers (optionally) in wheels
+* [ARROW-1022](https://issues.apache.org/jira/browse/ARROW-1022) - [Python] Add nthreads option to Feather read method
+* [ARROW-1024](https://issues.apache.org/jira/browse/ARROW-1024) - Python: Update build time numpy version to 1.10.1
+* [ARROW-1025](https://issues.apache.org/jira/browse/ARROW-1025) - [Website] Improve changelog on website
+* [ARROW-1027](https://issues.apache.org/jira/browse/ARROW-1027) - [Python] Allow negative indexing in fields/columns on pyarrow Table and Schema objects
+* [ARROW-1028](https://issues.apache.org/jira/browse/ARROW-1028) - [Python] Documentation updates after ARROW-1008
+* [ARROW-1029](https://issues.apache.org/jira/browse/ARROW-1029) - [Python] Fix --with-parquet build on Windows, add unit tests to Appveyor
+* [ARROW-1030](https://issues.apache.org/jira/browse/ARROW-1030) - Python: Account for library versioning in parquet-cpp
+* [ARROW-1031](https://issues.apache.org/jira/browse/ARROW-1031) - [GLib] Support pretty print
+* [ARROW-1037](https://issues.apache.org/jira/browse/ARROW-1037) - [GLib] Follow reader name change
+* [ARROW-1038](https://issues.apache.org/jira/browse/ARROW-1038) - [GLib] Follow writer name change
+* [ARROW-1040](https://issues.apache.org/jira/browse/ARROW-1040) - [GLib] Follow tensor IO
+* [ARROW-1044](https://issues.apache.org/jira/browse/ARROW-1044) - [GLib] Support Feather
+* [ARROW-182](https://issues.apache.org/jira/browse/ARROW-182) - [C++] Remove Array::Validate virtual function and make a separate method
+* [ARROW-29](https://issues.apache.org/jira/browse/ARROW-29) - C++: Add re2 as optional 3rd-party toolchain dependency
+* [ARROW-446](https://issues.apache.org/jira/browse/ARROW-446) - [Python] Document NativeFile interfaces, HDFS client in Sphinx
+* [ARROW-482](https://issues.apache.org/jira/browse/ARROW-482) - [Java] Provide API access to "custom_metadata" Field attribute in IPC setting
+* [ARROW-532](https://issues.apache.org/jira/browse/ARROW-532) - [Python] Expand pyarrow.parquet documentation for 0.3 release
+* [ARROW-629](https://issues.apache.org/jira/browse/ARROW-629) - [JS] Add unit test suite
+* [ARROW-714](https://issues.apache.org/jira/browse/ARROW-714) - [C++] Add import_pyarrow C API in the style of NumPy for thirdparty C++ users
+* [ARROW-819](https://issues.apache.org/jira/browse/ARROW-819) - [Python] Define public Cython API
+* [ARROW-872](https://issues.apache.org/jira/browse/ARROW-872) - [JS] Read streaming format
+* [ARROW-873](https://issues.apache.org/jira/browse/ARROW-873) - [JS] Implement fixed width list type
+* [ARROW-874](https://issues.apache.org/jira/browse/ARROW-874) - [JS] Read dictionary-encoded vectors
+* [ARROW-899](https://issues.apache.org/jira/browse/ARROW-899) - [Docs] Add CHANGELOG for 0.3.0
+* [ARROW-901](https://issues.apache.org/jira/browse/ARROW-901) - [Python] Write FixedSizeBinary to Parquet
+* [ARROW-923](https://issues.apache.org/jira/browse/ARROW-923) - [Docs] Generate Changelog for website with JIRA links
+* [ARROW-929](https://issues.apache.org/jira/browse/ARROW-929) - Move KEYS file to SVN, remove from git
+* [ARROW-943](https://issues.apache.org/jira/browse/ARROW-943) - [GLib] Support running unit tests with source archive
+* [ARROW-945](https://issues.apache.org/jira/browse/ARROW-945) - [GLib] Add a Lua example to show Torch integration
+* [ARROW-946](https://issues.apache.org/jira/browse/ARROW-946) - [GLib] Use "new" instead of "open" for constructor name
+* [ARROW-947](https://issues.apache.org/jira/browse/ARROW-947) - [Python] Improve execution time of manylinux1 build
+* [ARROW-953](https://issues.apache.org/jira/browse/ARROW-953) - Use cmake / curl from conda-forge in CI builds
+* [ARROW-954](https://issues.apache.org/jira/browse/ARROW-954) - Make it possible to compile Arrow with header-only boost
+* [ARROW-956](https://issues.apache.org/jira/browse/ARROW-956) - remove pandas pre-0.20.0 compat
+* [ARROW-957](https://issues.apache.org/jira/browse/ARROW-957) - [Doc] Add HDFS and Windows documents to doxygen output
+* [ARROW-961](https://issues.apache.org/jira/browse/ARROW-961) - [Python] Rename InMemoryOutputStream to BufferOutputStream
+* [ARROW-963](https://issues.apache.org/jira/browse/ARROW-963) - [GLib] Add equal
+* [ARROW-967](https://issues.apache.org/jira/browse/ARROW-967) - [GLib] Support initializing array with buffer
+* [ARROW-970](https://issues.apache.org/jira/browse/ARROW-970) - [Python] Accidentally calling pyarrow.Table() should not segfault process
+* [ARROW-977](https://issues.apache.org/jira/browse/ARROW-977) - [java] Add Timezone aware timestamp vectors
+* [ARROW-984](https://issues.apache.org/jira/browse/ARROW-984) - [GLib] Add Go examples
+* [ARROW-985](https://issues.apache.org/jira/browse/ARROW-985) - [GLib] Update package information
+* [ARROW-988](https://issues.apache.org/jira/browse/ARROW-988) - [JS] Add entry to Travis CI matrix
+* [ARROW-993](https://issues.apache.org/jira/browse/ARROW-993) - [GLib] Add missing error checks in Go examples
+* [ARROW-996](https://issues.apache.org/jira/browse/ARROW-996) - [Website] Add 0.3 release announce in Japanese
+* [ARROW-997](https://issues.apache.org/jira/browse/ARROW-997) - [Java] Implement transfer in FixedSizeListVector
+
+## Bug Fixes
+
+* [ARROW-1003](https://issues.apache.org/jira/browse/ARROW-1003) - [C++] Hdfs and java dlls fail to load when built for Windows with MSVC
+* [ARROW-1004](https://issues.apache.org/jira/browse/ARROW-1004) - ArrowInvalid: Invalid: Python object of type float is not None and is not a string, bool, or date object
+* [ARROW-1017](https://issues.apache.org/jira/browse/ARROW-1017) - Python: Table.to_pandas leaks memory
+* [ARROW-1023](https://issues.apache.org/jira/browse/ARROW-1023) - Python: Fix bundling of arrow-cpp for macOS
+* [ARROW-1033](https://issues.apache.org/jira/browse/ARROW-1033) - [Python] pytest discovers scripts/test_leak.py
+* [ARROW-1046](https://issues.apache.org/jira/browse/ARROW-1046) - [Python] Conform DataFrame metadata to pandas spec
+* [ARROW-1053](https://issues.apache.org/jira/browse/ARROW-1053) - [Python] Memory leak with RecordBatchFileReader
+* [ARROW-1054](https://issues.apache.org/jira/browse/ARROW-1054) - [Python] Test suite fails on pandas 0.19.2
+* [ARROW-813](https://issues.apache.org/jira/browse/ARROW-813) - [Python] setup.py sdist must also bundle dependent cmake modules
+* [ARROW-824](https://issues.apache.org/jira/browse/ARROW-824) - Date and Time Vectors should reflect timezone-less semantics
+* [ARROW-856](https://issues.apache.org/jira/browse/ARROW-856) - CmakeError by Unknown compiler.
+* [ARROW-881](https://issues.apache.org/jira/browse/ARROW-881) - [Python] Reconstruct Pandas DataFrame indexes using custom_metadata
+* [ARROW-909](https://issues.apache.org/jira/browse/ARROW-909) - libjemalloc.so.2: cannot open shared object file:
+* [ARROW-939](https://issues.apache.org/jira/browse/ARROW-939) - Fix division by zero for zero-dimensional Tensors
+* [ARROW-940](https://issues.apache.org/jira/browse/ARROW-940) - [JS] Generate multiple sets of artifacts
+* [ARROW-944](https://issues.apache.org/jira/browse/ARROW-944) - Python: Compat broken for pandas==0.18.1
+* [ARROW-948](https://issues.apache.org/jira/browse/ARROW-948) - [GLib] Update C++ header file list
+* [ARROW-952](https://issues.apache.org/jira/browse/ARROW-952) - Compilation error on macOS with clang-802.0.42
+* [ARROW-958](https://issues.apache.org/jira/browse/ARROW-958) - [Python] Conda build guide still needs ARROW_HOME, PARQUET_HOME
+* [ARROW-991](https://issues.apache.org/jira/browse/ARROW-991) - [Python] PyArray_SimpleNew should not be used with NPY_DATETIME
+* [ARROW-995](https://issues.apache.org/jira/browse/ARROW-995) - [Website] 0.3 release announce has a typo in reference
+* [ARROW-998](https://issues.apache.org/jira/browse/ARROW-998) - [Doc] File format documents incorrect schema location
+
+[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.4.0
+[6]: https://www.apache.org/dyn/closer.cgi/arrow/arrow-0.4.0/
+[8]: http://arrow.apache.org/blog/2017/05/23/0.4.0-release/
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/arrow/blob/8a700ccd/site/_release/index.md
----------------------------------------------------------------------
diff --git a/site/_release/index.md b/site/_release/index.md
index d422275..a7f9ab6 100644
--- a/site/_release/index.md
+++ b/site/_release/index.md
@@ -1,6 +1,6 @@
---
layout: default
-title: 0.3.0 Release
+title: Releases
permalink: /release/index.html
---
<!--
@@ -26,6 +26,7 @@ limitations under the License.
Navigate to the release page for downloads and the changelog.
+* [0.4.0 (22 May 2017)][4]
* [0.3.0 (5 May 2017)][1]
* [0.2.0 (18 February 2017)][2]
* [0.1.0 (10 October 2016)][3]
@@ -33,3 +34,4 @@ Navigate to the release page for downloads and the changelog.
[1]: {{ site.baseurl }}/release/0.3.0.html
[2]: {{ site.baseurl }}/release/0.2.0.html
[3]: {{ site.baseurl }}/release/0.1.0.html
+[4]: {{ site.baseurl }}/release/0.4.0.html
http://git-wip-us.apache.org/repos/asf/arrow/blob/8a700ccd/site/css/syntax.css
----------------------------------------------------------------------
diff --git a/site/css/syntax.css b/site/css/syntax.css
new file mode 100644
index 0000000..daf76ad
--- /dev/null
+++ b/site/css/syntax.css
@@ -0,0 +1,209 @@
+.highlight table td { padding: 5px; }
+.highlight table pre { margin: 0; }
+.highlight .cm {
+ color: #999988;
+ font-style: italic;
+}
+.highlight .cp {
+ color: #999999;
+ font-weight: bold;
+}
+.highlight .c1 {
+ color: #999988;
+ font-style: italic;
+}
+.highlight .cs {
+ color: #999999;
+ font-weight: bold;
+ font-style: italic;
+}
+.highlight .c, .highlight .cd {
+ color: #999988;
+ font-style: italic;
+}
+.highlight .err {
+ color: #a61717;
+ background-color: #e3d2d2;
+}
+.highlight .gd {
+ color: #000000;
+ background-color: #ffdddd;
+}
+.highlight .ge {
+ color: #000000;
+ font-style: italic;
+}
+.highlight .gr {
+ color: #aa0000;
+}
+.highlight .gh {
+ color: #999999;
+}
+.highlight .gi {
+ color: #000000;
+ background-color: #ddffdd;
+}
+.highlight .go {
+ color: #888888;
+}
+.highlight .gp {
+ color: #555555;
+}
+.highlight .gs {
+ font-weight: bold;
+}
+.highlight .gu {
+ color: #aaaaaa;
+}
+.highlight .gt {
+ color: #aa0000;
+}
+.highlight .kc {
+ color: #000000;
+ font-weight: bold;
+}
+.highlight .kd {
+ color: #000000;
+ font-weight: bold;
+}
+.highlight .kn {
+ color: #000000;
+ font-weight: bold;
+}
+.highlight .kp {
+ color: #000000;
+ font-weight: bold;
+}
+.highlight .kr {
+ color: #000000;
+ font-weight: bold;
+}
+.highlight .kt {
+ color: #445588;
+ font-weight: bold;
+}
+.highlight .k, .highlight .kv {
+ color: #000000;
+ font-weight: bold;
+}
+.highlight .mf {
+ color: #009999;
+}
+.highlight .mh {
+ color: #009999;
+}
+.highlight .il {
+ color: #009999;
+}
+.highlight .mi {
+ color: #009999;
+}
+.highlight .mo {
+ color: #009999;
+}
+.highlight .m, .highlight .mb, .highlight .mx {
+ color: #009999;
+}
+.highlight .sb {
+ color: #d14;
+}
+.highlight .sc {
+ color: #d14;
+}
+.highlight .sd {
+ color: #d14;
+}
+.highlight .s2 {
+ color: #d14;
+}
+.highlight .se {
+ color: #d14;
+}
+.highlight .sh {
+ color: #d14;
+}
+.highlight .si {
+ color: #d14;
+}
+.highlight .sx {
+ color: #d14;
+}
+.highlight .sr {
+ color: #009926;
+}
+.highlight .s1 {
+ color: #d14;
+}
+.highlight .ss {
+ color: #990073;
+}
+.highlight .s {
+ color: #d14;
+}
+.highlight .na {
+ color: #008080;
+}
+.highlight .bp {
+ color: #999999;
+}
+.highlight .nb {
+ color: #0086B3;
+}
+.highlight .nc {
+ color: #445588;
+ font-weight: bold;
+}
+.highlight .no {
+ color: #008080;
+}
+.highlight .nd {
+ color: #3c5d5d;
+ font-weight: bold;
+}
+.highlight .ni {
+ color: #800080;
+}
+.highlight .ne {
+ color: #990000;
+ font-weight: bold;
+}
+.highlight .nf {
+ color: #990000;
+ font-weight: bold;
+}
+.highlight .nl {
+ color: #990000;
+ font-weight: bold;
+}
+.highlight .nn {
+ color: #555555;
+}
+.highlight .nt {
+ color: #000080;
+}
+.highlight .vc {
+ color: #008080;
+}
+.highlight .vg {
+ color: #008080;
+}
+.highlight .vi {
+ color: #008080;
+}
+.highlight .nv {
+ color: #008080;
+}
+.highlight .ow {
+ color: #000000;
+ font-weight: bold;
+}
+.highlight .o {
+ color: #000000;
+ font-weight: bold;
+}
+.highlight .w {
+ color: #bbbbbb;
+}
+.highlight {
+ background-color: #f8f8f8;
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/8a700ccd/site/index.html
----------------------------------------------------------------------
diff --git a/site/index.html b/site/index.html
index d80925c..620b751 100644
--- a/site/index.html
+++ b/site/index.html
@@ -7,10 +7,10 @@ layout: default
<p class="lead">Powering Columnar In-Memory Analytics</p>
<p>
<a class="btn btn-lg btn-success" href="mailto:dev-subscribe@arrow.apache.org" role="button">Join Mailing List</a>
- <a class="btn btn-lg btn-primary" href="{{ site.baseurl }}/install/" role="button">Install (0.3.0 Release - May 5, 2017)</a>
+ <a class="btn btn-lg btn-primary" href="{{ site.baseurl }}/install/" role="button">Install (0.4.0 Release - May 22, 2017)</a>
</p>
</div>
- <h4>Latest News: <a href="{{ site.baseurl }}/blog/">Apache Arrow 0.3.0 release</a></h4>
+ <h4><strong>Latest News</strong>: <a href="{{ site.baseurl }}/blog/">Apache Arrow 0.4.0 release</a></h4>
<div class="row">
<div class="col-lg-4">
<h2>Fast</h2>
@@ -26,6 +26,12 @@ layout: default
</div>
</div> <!-- close "row" div -->
+<h2>Zero-Copy IPC and Streaming Messaging</h2>
+<div align="left">
+ <h4>Apache Arrow supports zero-copy shared memory IPC and a streaming wire
+ format that fully avoids traditional data serialization costs</h4>
+</div>
+
<h2>Performance Advantage of Columnar In-Memory</h2>
<div align="center">
<img src="img/simd.png" alt="SIMD" style="width:60%" />
http://git-wip-us.apache.org/repos/asf/arrow/blob/8a700ccd/site/install.md
----------------------------------------------------------------------
diff --git a/site/install.md b/site/install.md
index a575a1e..9018c6c 100644
--- a/site/install.md
+++ b/site/install.md
@@ -20,20 +20,17 @@ limitations under the License.
{% endcomment %}
-->
-## Current Version: 0.3.0
+## Current Version: 0.4.0
-### Released: 5 May 2017
+### Released: 22 May 2017
-Apache Arrow 0.3.0 is the third major release of the project and has seen
-significant iteration and hardening of logical types and the binary formats. It
-is safe for production use, though there may be API changes and binary format
-breaks in the future.
+See the [release notes][10] and [blog post][11] for more about what's new.
### Source release
-* **Source Release**: [apache-arrow-0.3.0.tar.gz][6]
+* **Source Release**: [apache-arrow-0.4.0.tar.gz][6]
* **Verification**: [md5][3], [asc][7]
-* [Git tag d8db8f8][2]
+* [Git tag a8f8ba0][2]
### Java Packages
@@ -132,12 +129,14 @@ These repositories are managed at
[red-data-tools/arrow-packages][9]. If you have any feedback, please
send it to the project instead of Apache Arrow project.
-[1]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0
-[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.3.0
-[3]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz.md5
-[4]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.3.0%22
+[1]: https://www-us.apache.org/dist/arrow/arrow-0.4.0/
+[2]: https://github.com/apache/arrow/releases/tag/apache-arrow-0.4.0
+[3]: https://www-us.apache.org/dist/arrow/arrow-0.4.0/apache-arrow-0.4.0.tar.gz.md5
+[4]: http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.arrow%22%20AND%20v%3A%220.4.0%22
[5]: http://conda-forge.github.io
-[6]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz
-[7]: https://dist.apache.org/repos/dist/release/arrow/arrow-0.3.0/apache-arrow-0.3.0.tar.gz.asc
+[6]: https://www-us.apache.org/dist/arrow/arrow-0.4.0/apache-arrow-0.4.0.tar.gz
+[7]: https://www-us.apache.org/dist/arrow/arrow-0.4.0/apache-arrow-0.4.0.tar.gz.asc
[8]: https://github.com/red-data-tools/parquet-glib
[9]: https://github.com/red-data-tools/arrow-packages
+[10]: http://arrow.apache.org/release/0.4.0.html
+[11]: http://arrow.apache.org/blog/2017/05/23/0.4.0-release/
\ No newline at end of file
[09/14] arrow git commit: ARROW-1057: Fix cmake warning and msvc
debug asserts
Posted by we...@apache.org.
ARROW-1057: Fix cmake warning and msvc debug asserts
Author: rip.nsk <ri...@gmail.com>
Closes #706 from rip-nsk/ARROW-1057 and squashes the following commits:
9bfe3199 [rip.nsk] Fix dependency of "arrow_objlib".
0c1e75d3 [rip.nsk] Fix "string iterator not dereferencable" assert
0b842c09 [rip.nsk] Fix "vector subscript out of range" assert
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/84b7ee18
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/84b7ee18
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/84b7ee18
Branch: refs/heads/master
Commit: 84b7ee18e0cef288f74fa8e9d8825306daea850a
Parents: d2cc199
Author: rip.nsk <ri...@gmail.com>
Authored: Tue May 23 14:06:32 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Wed May 31 13:45:48 2017 -0400
----------------------------------------------------------------------
cpp/CMakeLists.txt | 3 +--
cpp/src/arrow/array-decimal-test.cc | 3 +--
cpp/src/arrow/util/decimal.cc | 4 ++--
3 files changed, 4 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/84b7ee18/cpp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index 0ad7ef5..d5483f2 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -949,10 +949,9 @@ add_subdirectory(src/arrow)
add_subdirectory(src/arrow/io)
if (ARROW_IPC)
add_subdirectory(src/arrow/ipc)
+ set(ARROW_DEPENDENCIES ${ARROW_DEPENDENCIES} metadata_fbs)
endif()
-set(ARROW_DEPENDENCIES ${ARROW_DEPENDENCIES} metadata_fbs)
-
set(ARROW_SRCS
src/arrow/array.cc
src/arrow/buffer.cc
http://git-wip-us.apache.org/repos/asf/arrow/blob/84b7ee18/cpp/src/arrow/array-decimal-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/array-decimal-test.cc b/cpp/src/arrow/array-decimal-test.cc
index 4bde7ab..fb4b8d9 100644
--- a/cpp/src/arrow/array-decimal-test.cc
+++ b/cpp/src/arrow/array-decimal-test.cc
@@ -75,8 +75,7 @@ class DecimalTest : public DecimalTestBase<T> {
public:
std::vector<uint8_t> data(
const std::vector<T>& input, size_t byte_width) const override {
- std::vector<uint8_t> result;
- result.reserve(input.size() * byte_width);
+ std::vector<uint8_t> result(input.size() * byte_width);
// TODO(phillipc): There's probably a better way to do this
constexpr static const size_t bytes_per_element = sizeof(T);
for (size_t i = 0, j = 0; i < input.size(); ++i, j += bytes_per_element) {
http://git-wip-us.apache.org/repos/asf/arrow/blob/84b7ee18/cpp/src/arrow/util/decimal.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index 3d9fbd3..734df13 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -29,8 +29,8 @@ ARROW_EXPORT Status FromString(
}
int8_t sign = 1;
- auto charp = s.cbegin();
- auto end = s.cend();
+ auto charp = s.c_str();
+ auto end = charp + s.length();
if (*charp == '+' || *charp == '-') {
if (*charp == '-') { sign = -1; }
[07/14] arrow git commit: ARROW-1049: [java] vector template cleanup
Posted by we...@apache.org.
ARROW-1049: [java] vector template cleanup
Author: Julien Le Dem <ju...@apache.org>
Closes #703 from julienledem/template_cleanup and squashes the following commits:
d67cb7e [Julien Le Dem] remove duplicated code
f17e8cd [Julien Le Dem] template cleanup
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/b06602d0
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/b06602d0
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/b06602d0
Branch: refs/heads/master
Commit: b06602d08f5e5b741559449525edaa404bf1675c
Parents: a6e77f4
Author: Julien Le Dem <ju...@apache.org>
Authored: Mon May 22 16:06:33 2017 -0700
Committer: Wes McKinney <we...@twosigma.com>
Committed: Wed May 31 13:45:48 2017 -0400
----------------------------------------------------------------------
.../src/main/codegen/data/ValueVectorTypes.tdd | 2 +-
.../codegen/templates/AbstractFieldWriter.java | 6 -
.../AbstractPromotableFieldWriter.java | 6 -
.../main/codegen/templates/ComplexWriters.java | 11 -
.../codegen/templates/FixedValueVectors.java | 224 ++++---------------
.../codegen/templates/HolderReaderImpl.java | 134 +++--------
.../codegen/templates/NullableValueVectors.java | 31 +--
.../main/codegen/templates/UnionListWriter.java | 35 ++-
.../src/main/codegen/templates/UnionReader.java | 41 ++--
.../src/main/codegen/templates/UnionVector.java | 68 +++---
.../src/main/codegen/templates/UnionWriter.java | 32 +--
.../main/codegen/templates/ValueHolders.java | 2 +-
.../templates/VariableLengthVectors.java | 11 +-
13 files changed, 185 insertions(+), 418 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/b06602d0/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
index 4d8c99e..970d887 100644
--- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
+++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
@@ -118,7 +118,7 @@
maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal",
typeParams: [ {name: "scale", type: "int"}, { name: "precision", type: "int"}],
arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Decimal",
- fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}, {name: "scale", type: "int", include: false}, {name: "precision", type: "int", include: false}]
+ fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}]
}
]
},
http://git-wip-us.apache.org/repos/asf/arrow/blob/b06602d0/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
index 65bcc05..da8e4f5 100644
--- a/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractFieldWriter.java
@@ -58,15 +58,9 @@ abstract class AbstractFieldWriter extends AbstractBaseWriter implements FieldWr
fail("${name}");
}
- <#if minor.class == "Decimal">
- public void write${minor.class}(int start, ArrowBuf buffer) {
- fail("${name}");
- }
- <#else>
public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
fail("${name}");
}
- </#if>
</#list></#list>
http://git-wip-us.apache.org/repos/asf/arrow/blob/b06602d0/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
index 636b305..228c2c5 100644
--- a/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
+++ b/java/vector/src/main/codegen/templates/AbstractPromotableFieldWriter.java
@@ -78,15 +78,9 @@ abstract class AbstractPromotableFieldWriter extends AbstractFieldWriter {
getWriter(MinorType.${name?upper_case}).write(holder);
}
- <#if minor.class == "Decimal">
- public void write${minor.class}(int start, ArrowBuf buffer) {
- getWriter(MinorType.${name?upper_case}).write${minor.class}(start, buffer);
- }
- <#else>
public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
getWriter(MinorType.${name?upper_case}).write${minor.class}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
}
- </#if>
</#list></#list>
public void writeNull() {
http://git-wip-us.apache.org/repos/asf/arrow/blob/b06602d0/java/vector/src/main/codegen/templates/ComplexWriters.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java
index c23b89d..8ebecf3 100644
--- a/java/vector/src/main/codegen/templates/ComplexWriters.java
+++ b/java/vector/src/main/codegen/templates/ComplexWriters.java
@@ -111,17 +111,10 @@ public class ${eName}WriterImpl extends AbstractFieldWriter {
vector.getMutator().setValueCount(idx()+1);
}
- <#if minor.class == "Decimal">
- public void writeDecimal(int start, ArrowBuf buffer) {
- mutator.setSafe(idx(), 1, start, buffer);
- vector.getMutator().setValueCount(idx()+1);
- }
- <#else>
public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
mutator.setSafe(idx()<#if mode == "Nullable">, 1</#if><#list fields as field><#if field.include!true >, ${field.name}</#if></#list>);
vector.getMutator().setValueCount(idx()+1);
}
- </#if>
<#if mode == "Nullable">
@@ -146,11 +139,7 @@ package org.apache.arrow.vector.complex.writer;
public interface ${eName}Writer extends BaseWriter {
public void write(${minor.class}Holder h);
- <#if minor.class == "Decimal">
- public void writeDecimal(int start, ArrowBuf buffer);
- <#else>
public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>);
- </#if>
}
</#list>
http://git-wip-us.apache.org/repos/asf/arrow/blob/b06602d0/java/vector/src/main/codegen/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
index 5d80b66..395417b 100644
--- a/java/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -56,7 +56,7 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
private int allocationMonitor = 0;
<#if minor.typeParams??>
- <#list minor.typeParams as typeParam>
+ <#list minor.typeParams as typeParam>
private final ${typeParam.type} ${typeParam.name};
</#list>
@@ -67,7 +67,6 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
</#list>
}
<#else>
-
public ${className}(String name, BufferAllocator allocator) {
super(name, allocator);
}
@@ -166,7 +165,7 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
allocationMonitor = 0;
zeroVector();
super.reset();
- }
+ }
private void allocateBytes(final long size) {
if (size > MAX_ALLOCATION_SIZE) {
@@ -180,11 +179,11 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
allocationSizeInBytes = curSize;
}
-/**
- * Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one
- *
- * @throws org.apache.arrow.memory.OutOfMemoryException if it can't allocate the new buffer
- */
+ /**
+ * Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one
+ *
+ * @throws org.apache.arrow.memory.OutOfMemoryException if it can't allocate the new buffer
+ */
public void reAlloc() {
final long newAllocationSize = allocationSizeInBytes * 2L;
if (newAllocationSize > MAX_ALLOCATION_SIZE) {
@@ -311,76 +310,12 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
}
<#if (type.width > 8 || minor.class == "IntervalDay")>
-
public ${minor.javaType!type.javaType} get(int index) {
return data.slice(index * ${type.width}, ${type.width});
}
- <#if (minor.class == "Interval")>
- public void get(int index, ${minor.class}Holder holder){
-
- final int offsetIndex = index * ${type.width};
- holder.months = data.getInt(offsetIndex);
- holder.days = data.getInt(offsetIndex + ${minor.daysOffset});
- holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
- }
-
- public void get(int index, Nullable${minor.class}Holder holder){
- final int offsetIndex = index * ${type.width};
- holder.isSet = 1;
- holder.months = data.getInt(offsetIndex);
- holder.days = data.getInt(offsetIndex + ${minor.daysOffset});
- holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
- }
-
- @Override
- public ${friendlyType} getObject(int index) {
- final int offsetIndex = index * ${type.width};
- final int months = data.getInt(offsetIndex);
- final int days = data.getInt(offsetIndex + ${minor.daysOffset});
- final int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
- final Period p = new Period();
- return p.plusMonths(months).plusDays(days).plusMillis(millis);
- }
-
- public StringBuilder getAsStringBuilder(int index) {
-
- final int offsetIndex = index * ${type.width};
-
- int months = data.getInt(offsetIndex);
- final int days = data.getInt(offsetIndex + ${minor.daysOffset});
- int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
-
- final int years = (months / org.apache.arrow.vector.util.DateUtility.yearsToMonths);
- months = (months % org.apache.arrow.vector.util.DateUtility.yearsToMonths);
-
- final int hours = millis / (org.apache.arrow.vector.util.DateUtility.hoursToMillis);
- millis = millis % (org.apache.arrow.vector.util.DateUtility.hoursToMillis);
-
- final int minutes = millis / (org.apache.arrow.vector.util.DateUtility.minutesToMillis);
- millis = millis % (org.apache.arrow.vector.util.DateUtility.minutesToMillis);
-
- final long seconds = millis / (org.apache.arrow.vector.util.DateUtility.secondsToMillis);
- millis = millis % (org.apache.arrow.vector.util.DateUtility.secondsToMillis);
-
- final String yearString = (Math.abs(years) == 1) ? " year " : " years ";
- final String monthString = (Math.abs(months) == 1) ? " month " : " months ";
- final String dayString = (Math.abs(days) == 1) ? " day " : " days ";
-
-
- return(new StringBuilder().
- append(years).append(yearString).
- append(months).append(monthString).
- append(days).append(dayString).
- append(hours).append(":").
- append(minutes).append(":").
- append(seconds).append(".").
- append(millis));
- }
-
- <#elseif (minor.class == "IntervalDay")>
+ <#if (minor.class == "IntervalDay")>
public void get(int index, ${minor.class}Holder holder){
-
final int offsetIndex = index * ${type.width};
holder.days = data.getInt(offsetIndex);
holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
@@ -402,7 +337,6 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
return p.plusDays(days).plusMillis(millis);
}
-
public StringBuilder getAsStringBuilder(int index) {
final int offsetIndex = index * ${type.width};
@@ -428,8 +362,7 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
append(millis));
}
- <#elseif minor.class == "Decimal">
-
+ <#elseif minor.class == "Decimal">
public void get(int index, ${minor.class}Holder holder) {
holder.start = index * ${type.width};
holder.buffer = data;
@@ -450,7 +383,7 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromArrowBuf(data, ${type.width} * index, scale);
}
- <#else>
+ <#else>
public void get(int index, ${minor.class}Holder holder){
holder.buffer = data;
holder.start = index * ${type.width};
@@ -467,44 +400,35 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
return data.slice(index * ${type.width}, ${type.width})
}
- </#if>
+ </#if>
<#else> <#-- type.width <= 8 -->
public ${minor.javaType!type.javaType} get(int index) {
return data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
}
-
- <#if type.width == 4>
+ <#if type.width == 4>
public long getTwoAsLong(int index) {
return data.getLong(index * ${type.width});
}
- </#if>
-
- <#if minor.class == "DateDay" ||
- minor.class == "TimeSec" ||
- minor.class == "TimeMicro" ||
- minor.class == "TimeNano">
+ </#if>
+ <#if minor.class == "DateDay" ||
+ minor.class == "TimeSec" ||
+ minor.class == "TimeMicro" ||
+ minor.class == "TimeNano">
@Override
public ${friendlyType} getObject(int index) {
return get(index);
}
- <#elseif minor.class == "DateMilli">
+ <#elseif minor.class == "DateMilli" || minor.class == "TimeMilli" || minor.class == "TimeStampMilli">
@Override
public ${friendlyType} getObject(int index) {
- org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
- return date;
+ org.joda.time.LocalDateTime ldt = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
+ return ldt;
}
- <#elseif minor.class == "TimeMilli">
- @Override
- public ${friendlyType} getObject(int index) {
- org.joda.time.LocalDateTime time = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
- return time;
- }
-
- <#elseif minor.class == "TimeStampSec">
+ <#elseif minor.class == "TimeStampSec">
@Override
public ${friendlyType} getObject(int index) {
long secs = java.util.concurrent.TimeUnit.SECONDS.toMillis(get(index));
@@ -512,14 +436,7 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
return date;
}
- <#elseif minor.class == "TimeStampMilli">
- @Override
- public ${friendlyType} getObject(int index) {
- org.joda.time.LocalDateTime date = new org.joda.time.LocalDateTime(get(index), org.joda.time.DateTimeZone.UTC);
- return date;
- }
-
- <#elseif minor.class == "TimeStampMicro">
+ <#elseif minor.class == "TimeStampMicro">
@Override
public ${friendlyType} getObject(int index) {
// value is truncated when converting microseconds to milliseconds in order to use DateTime type
@@ -528,7 +445,7 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
return date;
}
- <#elseif minor.class == "TimeStampNano">
+ <#elseif minor.class == "TimeStampNano">
@Override
public ${friendlyType} getObject(int index) {
// value is truncated when converting nanoseconds to milliseconds in order to use DateTime type
@@ -537,7 +454,7 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
return date;
}
- <#elseif minor.class == "IntervalYear">
+ <#elseif minor.class == "IntervalYear">
@Override
public ${friendlyType} getObject(int index) {
@@ -550,7 +467,6 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
}
public StringBuilder getAsStringBuilder(int index) {
-
int months = data.getInt(index);
final int years = (months / org.apache.arrow.vector.util.DateUtility.yearsToMonths);
@@ -564,15 +480,7 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
append(months).append(monthString));
}
- <#elseif minor.class == "Decimal9" || minor.class == "Decimal18">
- @Override
- public ${friendlyType} getObject(int index) {
-
- final BigInteger value = BigInteger.valueOf(((${type.boxedType})get(index)).${type.javaType}Value());
- return new BigDecimal(value, getField().getScale());
- }
-
- <#else>
+ <#else>
@Override
public ${friendlyType} getObject(int index) {
return get(index);
@@ -580,14 +488,9 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
public ${minor.javaType!type.javaType} getPrimitiveObject(int index) {
return get(index);
}
- </#if>
- public void get(int index, ${minor.class}Holder holder){
- <#if minor.class.startsWith("Decimal")>
- holder.scale = getField().getScale();
- holder.precision = getField().getPrecision();
</#if>
-
+ public void get(int index, ${minor.class}Holder holder){
holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
}
@@ -596,22 +499,22 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
}
+ </#if> <#-- type.width -->
+ }
- </#if> <#-- type.width -->
- }
-
- /**
- * ${minor.class}.Mutator implements a mutable vector of fixed width values. Elements in the
- * vector are accessed by position from the logical start of the vector. Values should be pushed
- * onto the vector sequentially, but may be randomly accessed.
- * The width of each element is ${type.width} byte(s)
- * The equivalent Java primitive is '${minor.javaType!type.javaType}'
- *
- * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
- */
+ /**
+ * ${minor.class}.Mutator implements a mutable vector of fixed width values. Elements in the
+ * vector are accessed by position from the logical start of the vector. Values should be pushed
+ * onto the vector sequentially, but may be randomly accessed.
+ * The width of each element is ${type.width} byte(s)
+ * The equivalent Java primitive is '${minor.javaType!type.javaType}'
+ *
+ * NB: this class is automatically generated from FixedValueVectorTypes.tdd using FreeMarker.
+ */
public final class Mutator extends BaseDataValueVector.BaseMutator {
private Mutator(){};
+
/**
* Set the element at the given index to the given value. Note that widths smaller than
* 32 bits are handled by the ArrowBuf interface.
@@ -619,7 +522,7 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
* @param index position of the bit to set
* @param value value to set
*/
- <#if (type.width > 8) || minor.class == "IntervalDay">
+ <#if (type.width > 8) || minor.class == "IntervalDay">
public void set(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
data.setBytes(index * ${type.width}, value, 0, ${type.width});
}
@@ -631,38 +534,7 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
data.setBytes(index * ${type.width}, value, 0, ${type.width});
}
- <#if (minor.class == "Interval")>
- public void set(int index, int months, int days, int milliseconds){
- final int offsetIndex = index * ${type.width};
- data.setInt(offsetIndex, months);
- data.setInt((offsetIndex + ${minor.daysOffset}), days);
- data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds);
- }
-
- protected void set(int index, ${minor.class}Holder holder){
- set(index, holder.months, holder.days, holder.milliseconds);
- }
-
- protected void set(int index, Nullable${minor.class}Holder holder){
- set(index, holder.months, holder.days, holder.milliseconds);
- }
-
- public void setSafe(int index, int months, int days, int milliseconds){
- while(index >= getValueCapacity()) {
- reAlloc();
- }
- set(index, months, days, milliseconds);
- }
-
- public void setSafe(int index, Nullable${minor.class}Holder holder){
- setSafe(index, holder.months, holder.days, holder.milliseconds);
- }
-
- public void setSafe(int index, ${minor.class}Holder holder){
- setSafe(index, holder.months, holder.days, holder.milliseconds);
- }
-
- <#elseif (minor.class == "IntervalDay")>
+ <#if (minor.class == "IntervalDay")>
public void set(int index, int days, int milliseconds){
final int offsetIndex = index * ${type.width};
data.setInt(offsetIndex, days);
@@ -672,6 +544,7 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
protected void set(int index, ${minor.class}Holder holder){
set(index, holder.days, holder.milliseconds);
}
+
protected void set(int index, Nullable${minor.class}Holder holder){
set(index, holder.days, holder.milliseconds);
}
@@ -691,8 +564,7 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
setSafe(index, holder.days, holder.milliseconds);
}
- <#elseif minor.class == "Decimal">
-
+ <#elseif minor.class == "Decimal">
public void set(int index, ${minor.class}Holder holder){
set(index, holder.start, holder.buffer);
}
@@ -719,8 +591,7 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
data.setBytes(index * ${type.width}, buffer, start, ${type.width});
}
- <#else>
-
+ <#else>
protected void set(int index, ${minor.class}Holder holder){
set(index, holder.start, holder.buffer);
}
@@ -736,6 +607,7 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
public void setSafe(int index, ${minor.class}Holder holder){
setSafe(index, holder.start, holder.buffer);
}
+
public void setSafe(int index, Nullable${minor.class}Holder holder){
setSafe(index, holder.start, holder.buffer);
}
@@ -750,7 +622,7 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
public void set(int index, Nullable${minor.class}Holder holder){
data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width});
}
- </#if>
+ </#if>
@Override
public void generateTestData(int count) {
@@ -765,7 +637,7 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
}
}
- <#else> <#-- type.width <= 8 -->
+ <#else> <#-- type.width <= 8 -->
public void set(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, value);
}
@@ -826,7 +698,7 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
}
}
- </#if> <#-- type.width -->
+ </#if> <#-- type.width -->
@Override
public void setValueCount(int valueCount) {
@@ -846,6 +718,6 @@ public final class ${className} extends BaseDataValueVector implements FixedWidt
}
}
-</#if> <#-- type.major -->
+ </#if> <#-- type.major -->
</#list>
</#list>
http://git-wip-us.apache.org/repos/asf/arrow/blob/b06602d0/java/vector/src/main/codegen/templates/HolderReaderImpl.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/HolderReaderImpl.java b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
index e990fcc..471fe41 100644
--- a/java/vector/src/main/codegen/templates/HolderReaderImpl.java
+++ b/java/vector/src/main/codegen/templates/HolderReaderImpl.java
@@ -29,7 +29,7 @@
<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
<#assign safeType=friendlyType />
<#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
-<#assign fields = minor.fields!type.fields />
+<#assign fields = minor.fields!type.fields + minor.typeParams![]/>
<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/${holderMode}${name}HolderReaderImpl.java" />
<#include "/@includes/license.ftl" />
@@ -81,7 +81,6 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
<#else>
return true;
</#if>
-
}
@Override
@@ -101,121 +100,50 @@ public class ${holderMode}${name}HolderReaderImpl extends AbstractFieldReader {
// read friendly type
@Override
- public ${friendlyType} read${safeType}(){
-<#if nullMode == "Nullable">
+ public ${friendlyType} read${safeType}() {
+ <#if nullMode == "Nullable">
if (!isSet()) {
return null;
}
-</#if>
-
-<#if type.major == "VarLen">
-
- int length = holder.end - holder.start;
- byte[] value = new byte [length];
- holder.buffer.getBytes(holder.start, value, 0, length);
-
- <#if minor.class == "VarBinary">
- return value;
- <#elseif minor.class == "Var16Char">
- return new String(value);
- <#elseif minor.class == "VarChar">
- Text text = new Text();
- text.set(value);
- return text;
</#if>
-<#elseif minor.class == "Interval">
- Period p = new Period();
- return p.plusMonths(holder.months).plusDays(holder.days).plusMillis(holder.milliseconds);
-
-<#elseif minor.class == "IntervalDay">
- Period p = new Period();
- return p.plusDays(holder.days).plusMillis(holder.milliseconds);
-
-<#elseif minor.class == "Bit" >
- return new Boolean(holder.value != 0);
-<#elseif minor.class == "Decimal" >
- return (BigDecimal) readSingleObject();
-<#else>
- ${friendlyType} value = new ${friendlyType}(this.holder.value);
- return value;
-</#if>
-
+ <#if type.major == "VarLen">
+ int length = holder.end - holder.start;
+ byte[] value = new byte [length];
+ holder.buffer.getBytes(holder.start, value, 0, length);
+ <#if minor.class == "VarBinary">
+ return value;
+ <#elseif minor.class == "VarChar">
+ Text text = new Text();
+ text.set(value);
+ return text;
+ </#if>
+ <#elseif minor.class == "IntervalDay">
+ Period p = new Period();
+ return p.plusDays(holder.days).plusMillis(holder.milliseconds);
+ <#elseif minor.class == "Bit" >
+ return new Boolean(holder.value != 0);
+ <#elseif minor.class == "Decimal">
+ byte[] bytes = new byte[${type.width}];
+ holder.buffer.getBytes(holder.start, bytes, 0, ${type.width});
+ ${friendlyType} value = new BigDecimal(new BigInteger(bytes), holder.scale);
+ return value;
+ <#else>
+ ${friendlyType} value = new ${friendlyType}(this.holder.value);
+ return value;
+ </#if>
}
@Override
public Object readObject() {
- return readSingleObject();
+ return read${safeType}();
}
- private Object readSingleObject() {
-<#if nullMode == "Nullable">
- if (!isSet()) {
- return null;
- }
-</#if>
-
-<#if type.major == "VarLen">
- <#if minor.class != "Decimal">
- int length = holder.end - holder.start;
- byte[] value = new byte [length];
- holder.buffer.getBytes(holder.start, value, 0, length);
- </#if>
-
-<#if minor.class == "VarBinary">
- return value;
-<#elseif minor.class == "Var16Char">
- return new String(value);
-<#elseif minor.class == "VarChar">
- Text text = new Text();
- text.set(value);
- return text;
-<#elseif minor.class == "Decimal" >
- return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromArrowBuf(holder.buffer, holder.start, holder.scale);
-</#if>
-
-<#elseif minor.class == "Interval">
- Period p = new Period();
- return p.plusMonths(holder.months).plusDays(holder.days).plusMillis(holder.milliseconds);
-
-<#elseif minor.class == "IntervalDay">
- Period p = new Period();
- return p.plusDays(holder.days).plusMillis(holder.milliseconds);
-
-<#elseif minor.class == "Decimal28Dense" ||
- minor.class == "Decimal38Dense">
- return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromDense(holder.buffer,
- holder.start,
- holder.nDecimalDigits,
- holder.scale,
- holder.maxPrecision,
- holder.WIDTH);
-
-<#elseif minor.class == "Decimal28Sparse" ||
- minor.class == "Decimal38Sparse">
- return org.apache.arrow.vector.util.DecimalUtility.getBigDecimalFromSparse(holder.buffer,
- holder.start,
- holder.nDecimalDigits,
- holder.scale);
-
-<#elseif minor.class == "Bit" >
- return new Boolean(holder.value != 0);
-<#elseif minor.class == "Decimal">
- byte[] bytes = new byte[${type.width}];
- holder.buffer.getBytes(holder.start, bytes, 0, ${type.width});
- ${friendlyType} value = new BigDecimal(new BigInteger(bytes), holder.scale);
- return value;
-<#else>
- ${friendlyType} value = new ${friendlyType}(this.holder.value);
- return value;
-</#if>
- }
-
-<#if nullMode != "Nullable">
+ <#if nullMode != "Nullable">
public void copyAsValue(${minor.class?cap_first}Writer writer){
writer.write(holder);
}
-</#if>
+ </#if>
}
</#list>
http://git-wip-us.apache.org/repos/asf/arrow/blob/b06602d0/java/vector/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 3231c4c..dfd5755 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -470,11 +470,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type
public void get(int index, Nullable${minor.class}Holder holder){
vAccessor.get(index, holder);
holder.isSet = bAccessor.get(index);
-
- <#if minor.class.startsWith("Decimal")>
- holder.scale = scale;
- holder.precision = precision;
- </#if>
}
@Override
@@ -486,7 +481,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
}
}
- <#if minor.class == "Interval" || minor.class == "IntervalDay" || minor.class == "IntervalYear">
+ <#if minor.class == "Interval" || minor.class == "IntervalDay">
public StringBuilder getAsStringBuilder(int index) {
if (isNull(index)) {
return null;
@@ -628,7 +623,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
}
<#assign fields = minor.fields!type.fields />
- public void set(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ){
+ public void set(int index, int isSet<#list fields as field>, ${field.type} ${field.name}Field</#list> ){
final ${valuesName}.Mutator valuesMutator = values.getMutator();
<#if type.major == "VarLen">
for (int i = lastSet + 1; i < index; i++) {
@@ -644,7 +639,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type
<#if type.major == "VarLen">
fillEmpties(index);
</#if>
-
bits.getMutator().setSafe(index, isSet);
values.getMutator().setSafe(index<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>);
setCount++;
@@ -653,7 +647,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type
public void setSafe(int index, Nullable${minor.class}Holder value) {
-
<#if type.major == "VarLen">
fillEmpties(index);
</#if>
@@ -664,7 +657,6 @@ public final class ${className} extends BaseDataValueVector implements <#if type
}
public void setSafe(int index, ${minor.class}Holder value) {
-
<#if type.major == "VarLen">
fillEmpties(index);
</#if>
@@ -674,18 +666,17 @@ public final class ${className} extends BaseDataValueVector implements <#if type
<#if type.major == "VarLen">lastSet = index;</#if>
}
- <#if !(type.major == "VarLen" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense" || minor.class == "Interval" || minor.class == "IntervalDay")>
- public void setSafe(int index, ${minor.javaType!type.javaType} value) {
- <#if type.major == "VarLen">
- fillEmpties(index);
- </#if>
- bits.getMutator().setSafeToOne(index);
- values.getMutator().setSafe(index, value);
- setCount++;
- }
+ <#if !(type.major == "VarLen" || minor.class == "IntervalYear" || minor.class == "IntervalDay")>
+ public void setSafe(int index, ${minor.javaType!type.javaType} value) {
+ <#if type.major == "VarLen">
+ fillEmpties(index);
+ </#if>
+ bits.getMutator().setSafeToOne(index);
+ values.getMutator().setSafe(index, value);
+ setCount++;
+ }
</#if>
-
@Override
public void setValueCount(int valueCount) {
assert valueCount >= 0;
http://git-wip-us.apache.org/repos/asf/arrow/blob/b06602d0/java/vector/src/main/codegen/templates/UnionListWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java
index d019a1e..d6e2f9f 100644
--- a/java/vector/src/main/codegen/templates/UnionListWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionListWriter.java
@@ -89,27 +89,23 @@ public class UnionListWriter extends AbstractFieldWriter {
public void setPosition(int index) {
super.setPosition(index);
}
-
<#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
<#assign fields = minor.fields!type.fields />
<#assign uncappedName = name?uncap_first/>
-
+ <#if uncappedName == "int" ><#assign uncappedName = "integer" /></#if>
<#if !minor.typeParams?? >
@Override
- public ${name}Writer <#if uncappedName == "int">integer<#else>${uncappedName}</#if>() {
+ public ${name}Writer ${uncappedName}() {
return this;
}
@Override
- public ${name}Writer <#if uncappedName == "int">integer<#else>${uncappedName}</#if>(String name) {
-// assert inMap;
+ public ${name}Writer ${uncappedName}(String name) {
mapName = name;
- return writer.<#if uncappedName == "int">integer<#else>${uncappedName}</#if>(name);
+ return writer.${uncappedName}(name);
}
-
</#if>
-
</#list></#list>
@Override
@@ -149,33 +145,28 @@ public class UnionListWriter extends AbstractFieldWriter {
@Override
public void start() {
-// assert inMap;
writer.start();
}
@Override
public void end() {
-// if (inMap) {
writer.end();
inMap = false;
-// }
}
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign fields = minor.fields!type.fields />
- <#assign uncappedName = name?uncap_first/>
-
- <#if !minor.class?starts_with("Decimal")>
-
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ <#assign name = minor.class?cap_first />
+ <#assign fields = minor.fields!type.fields />
+ <#assign uncappedName = name?uncap_first/>
+ <#if !minor.typeParams?? >
@Override
public void write${name}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
-// assert !inMap;
writer.write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
writer.setPosition(writer.idx()+1);
}
- </#if>
-
- </#list></#list>
-
+ </#if>
+ </#list>
+ </#list>
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/b06602d0/java/vector/src/main/codegen/templates/UnionReader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/UnionReader.java b/java/vector/src/main/codegen/templates/UnionReader.java
index 0b5a209..2246fb3 100644
--- a/java/vector/src/main/codegen/templates/UnionReader.java
+++ b/java/vector/src/main/codegen/templates/UnionReader.java
@@ -79,13 +79,16 @@ public class UnionReader extends AbstractFieldReader {
return (FieldReader) getMap();
case LIST:
return (FieldReader) getList();
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign uncappedName = name?uncap_first/>
- <#if !minor.class?starts_with("Decimal")>
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ <#assign name = minor.class?cap_first />
+ <#assign uncappedName = name?uncap_first/>
+ <#if !minor.typeParams?? >
case ${name?upper_case}:
return (FieldReader) get${name}();
- </#if>
- </#list></#list>
+ </#if>
+ </#list>
+ </#list>
default:
throw new UnsupportedOperationException("Unsupported type: " + MinorType.values()[typeValue]);
}
@@ -140,14 +143,16 @@ public class UnionReader extends AbstractFieldReader {
return getReaderForIndex(idx()).size();
}
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign uncappedName = name?uncap_first/>
- <#assign boxedType = (minor.boxedType!type.boxedType) />
- <#assign javaType = (minor.javaType!type.javaType) />
- <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
- <#assign safeType=friendlyType />
- <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
- <#if !minor.class?starts_with("Decimal")>
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ <#assign name = minor.class?cap_first />
+ <#assign uncappedName = name?uncap_first/>
+ <#assign boxedType = (minor.boxedType!type.boxedType) />
+ <#assign javaType = (minor.javaType!type.javaType) />
+ <#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
+ <#assign safeType=friendlyType />
+ <#if safeType=="byte[]"><#assign safeType="ByteArray" /></#if>
+ <#if !minor.typeParams?? >
private ${name}ReaderImpl ${uncappedName}Reader;
@@ -167,8 +172,9 @@ public class UnionReader extends AbstractFieldReader {
public void copyAsValue(${name}Writer writer){
getReaderForIndex(idx()).copyAsValue(writer);
}
- </#if>
- </#list></#list>
+ </#if>
+ </#list>
+ </#list>
@Override
public void copyAsValue(ListWriter writer) {
@@ -184,7 +190,7 @@ public class UnionReader extends AbstractFieldReader {
}
}
}
-
+
public FieldReader reader(String name){
return getMap().reader(name);
}
@@ -197,6 +203,3 @@ public class UnionReader extends AbstractFieldReader {
return getReaderForIndex(idx()).next();
}
}
-
-
-
http://git-wip-us.apache.org/repos/asf/arrow/blob/b06602d0/java/vector/src/main/codegen/templates/UnionVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index ec42b12..aa9d34d 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -145,12 +145,13 @@ public class UnionVector implements FieldVector {
}
return mapVector;
}
-
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign fields = minor.fields!type.fields />
- <#assign uncappedName = name?uncap_first/>
- <#assign lowerCaseName = name?lower_case/>
- <#if !minor.class?starts_with("Decimal")>
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ <#assign name = minor.class?cap_first />
+ <#assign fields = minor.fields!type.fields />
+ <#assign uncappedName = name?uncap_first/>
+ <#assign lowerCaseName = name?lower_case/>
+ <#if !minor.typeParams?? >
private Nullable${name}Vector ${uncappedName}Vector;
@@ -167,10 +168,9 @@ public class UnionVector implements FieldVector {
}
return ${uncappedName}Vector;
}
-
- </#if>
-
- </#list></#list>
+ </#if>
+ </#list>
+ </#list>
public ListVector getList() {
if (listVector == null) {
@@ -401,22 +401,23 @@ public class UnionVector implements FieldVector {
public class Accessor extends BaseValueVector.BaseAccessor {
-
@Override
public Object getObject(int index) {
int type = typeVector.getAccessor().get(index);
switch (MinorType.values()[type]) {
case NULL:
return null;
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign fields = minor.fields!type.fields />
- <#assign uncappedName = name?uncap_first/>
- <#if !minor.class?starts_with("Decimal")>
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ <#assign name = minor.class?cap_first />
+ <#assign fields = minor.fields!type.fields />
+ <#assign uncappedName = name?uncap_first/>
+ <#if !minor.typeParams?? >
case ${name?upper_case}:
return get${name}Vector().getAccessor().getObject(index);
- </#if>
-
- </#list></#list>
+ </#if>
+ </#list>
+ </#list>
case MAP:
return getMap().getAccessor().getObject(index);
case LIST:
@@ -473,17 +474,20 @@ public class UnionVector implements FieldVector {
writer.setPosition(index);
MinorType type = reader.getMinorType();
switch (type) {
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign fields = minor.fields!type.fields />
- <#assign uncappedName = name?uncap_first/>
- <#if !minor.class?starts_with("Decimal")>
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ <#assign name = minor.class?cap_first />
+ <#assign fields = minor.fields!type.fields />
+ <#assign uncappedName = name?uncap_first/>
+ <#if !minor.typeParams?? >
case ${name?upper_case}:
Nullable${name}Holder ${uncappedName}Holder = new Nullable${name}Holder();
reader.read(${uncappedName}Holder);
setSafe(index, ${uncappedName}Holder);
break;
- </#if>
- </#list></#list>
+ </#if>
+ </#list>
+ </#list>
case MAP: {
ComplexCopier.copy(reader, writer);
break;
@@ -496,18 +500,20 @@ public class UnionVector implements FieldVector {
throw new UnsupportedOperationException();
}
}
-
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign fields = minor.fields!type.fields />
- <#assign uncappedName = name?uncap_first/>
- <#if !minor.class?starts_with("Decimal")>
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ <#assign name = minor.class?cap_first />
+ <#assign fields = minor.fields!type.fields />
+ <#assign uncappedName = name?uncap_first/>
+ <#if !minor.typeParams?? >
public void setSafe(int index, Nullable${name}Holder holder) {
setType(index, MinorType.${name?upper_case});
get${name}Vector().getMutator().setSafe(index, holder);
}
- </#if>
- </#list></#list>
+ </#if>
+ </#list>
+ </#list>
public void setType(int index, MinorType type) {
typeVector.getMutator().setSafe(index, (byte) type.ordinal());
http://git-wip-us.apache.org/repos/asf/arrow/blob/b06602d0/java/vector/src/main/codegen/templates/UnionWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java
index 4a7c472..f892bac 100644
--- a/java/vector/src/main/codegen/templates/UnionWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionWriter.java
@@ -117,26 +117,26 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter {
case LIST:
return getListWriter();
<#list vv.types as type>
- <#list type.minor as minor>
- <#assign name = minor.class?cap_first />
- <#assign fields = minor.fields!type.fields />
- <#assign uncappedName = name?uncap_first/>
- <#if !minor.typeParams??>
+ <#list type.minor as minor>
+ <#assign name = minor.class?cap_first />
+ <#assign fields = minor.fields!type.fields />
+ <#assign uncappedName = name?uncap_first/>
+ <#if !minor.typeParams??>
case ${name?upper_case}:
return get${name}Writer();
- </#if>
- </#list>
+ </#if>
+ </#list>
</#list>
default:
throw new UnsupportedOperationException("Unknown type: " + minorType);
}
}
-
- <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
- <#assign fields = minor.fields!type.fields />
- <#assign uncappedName = name?uncap_first/>
-
- <#if !minor.typeParams?? >
+ <#list vv.types as type>
+ <#list type.minor as minor>
+ <#assign name = minor.class?cap_first />
+ <#assign fields = minor.fields!type.fields />
+ <#assign uncappedName = name?uncap_first/>
+ <#if !minor.typeParams?? >
private ${name}Writer ${name?uncap_first}Writer;
@@ -166,9 +166,9 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter {
get${name}Writer().setPosition(idx());
get${name}Writer().write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
}
- </#if>
-
- </#list></#list>
+ </#if>
+ </#list>
+ </#list>
public void writeNull() {
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/b06602d0/java/vector/src/main/codegen/templates/ValueHolders.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/ValueHolders.java b/java/vector/src/main/codegen/templates/ValueHolders.java
index a474b69..b74d919 100644
--- a/java/vector/src/main/codegen/templates/ValueHolders.java
+++ b/java/vector/src/main/codegen/templates/ValueHolders.java
@@ -50,7 +50,7 @@ public final class ${className} implements ValueHolder{
<#if mode.name == "Optional">public int isSet;
<#else>public final int isSet = 1;</#if>
- <#assign fields = minor.fields!type.fields />
+ <#assign fields = (minor.fields!type.fields) + (minor.typeParams![]) />
<#list fields as field>
public ${field.type} ${field.name};
</#list>
http://git-wip-us.apache.org/repos/asf/arrow/blob/b06602d0/java/vector/src/main/codegen/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/VariableLengthVectors.java b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
index f13291b..2ad7d20 100644
--- a/java/vector/src/main/codegen/templates/VariableLengthVectors.java
+++ b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
@@ -421,23 +421,22 @@ public final class ${className} extends BaseDataValueVector implements VariableW
holder.buffer = data;
}
-
<#switch minor.class>
- <#case "VarChar">
+ <#case "VarChar">
@Override
public ${friendlyType} getObject(int index) {
Text text = new Text();
text.set(get(index));
return text;
}
- <#break>
- <#case "Decimal">
+ <#break>
+ <#case "Decimal">
@Override
public ${friendlyType} getObject(int index) {
return new BigDecimal(new BigInteger(get(index)), scale);
}
- <#break>
- <#default>
+ <#break>
+ <#default>
@Override
public ${friendlyType} getObject(int index) {
return get(index);
[02/14] arrow git commit: Increment version to 0.5.0-SNAPSHOT
Posted by we...@apache.org.
Increment version to 0.5.0-SNAPSHOT
Change-Id: I9b4a13830c41426ab728cdb2e2107c3537cd9ccf
Conflicts:
java/format/pom.xml
java/memory/pom.xml
java/pom.xml
java/tools/pom.xml
java/vector/pom.xml
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/cf4ef5ef
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/cf4ef5ef
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/cf4ef5ef
Branch: refs/heads/master
Commit: cf4ef5efb012a05ed7ce0086cb8136a1e5db412b
Parents: a8f8ba0
Author: Wes McKinney <we...@twosigma.com>
Authored: Mon May 22 20:31:50 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Wed May 31 13:45:34 2017 -0400
----------------------------------------------------------------------
java/format/pom.xml | 2 +-
java/memory/pom.xml | 2 +-
java/pom.xml | 4 ++--
java/tools/pom.xml | 2 +-
java/vector/pom.xml | 2 +-
5 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/cf4ef5ef/java/format/pom.xml
----------------------------------------------------------------------
diff --git a/java/format/pom.xml b/java/format/pom.xml
index 6efdf93..e092754 100644
--- a/java/format/pom.xml
+++ b/java/format/pom.xml
@@ -15,7 +15,7 @@
<parent>
<artifactId>arrow-java-root</artifactId>
<groupId>org.apache.arrow</groupId>
- <version>0.4.0</version>
+ <version>0.5.0-SNAPSHOT</version>
</parent>
<artifactId>arrow-format</artifactId>
http://git-wip-us.apache.org/repos/asf/arrow/blob/cf4ef5ef/java/memory/pom.xml
----------------------------------------------------------------------
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index bba043c..dc4d0da 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -14,7 +14,7 @@
<parent>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.4.0</version>
+ <version>0.5.0-SNAPSHOT</version>
</parent>
<artifactId>arrow-memory</artifactId>
<name>Arrow Memory</name>
http://git-wip-us.apache.org/repos/asf/arrow/blob/cf4ef5ef/java/pom.xml
----------------------------------------------------------------------
diff --git a/java/pom.xml b/java/pom.xml
index 8237a30..1ec3d56 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -20,7 +20,7 @@
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.4.0</version>
+ <version>0.5.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Arrow Java Root POM</name>
@@ -42,7 +42,7 @@
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</connection>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/arrow.git</developerConnection>
<url>https://github.com/apache/arrow</url>
- <tag>apache-arrow-0.4.0</tag>
+ <tag>HEAD</tag>
</scm>
<mailingLists>
http://git-wip-us.apache.org/repos/asf/arrow/blob/cf4ef5ef/java/tools/pom.xml
----------------------------------------------------------------------
diff --git a/java/tools/pom.xml b/java/tools/pom.xml
index 31f931b..53331f2 100644
--- a/java/tools/pom.xml
+++ b/java/tools/pom.xml
@@ -14,7 +14,7 @@
<parent>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.4.0</version>
+ <version>0.5.0-SNAPSHOT</version>
</parent>
<artifactId>arrow-tools</artifactId>
<name>Arrow Tools</name>
http://git-wip-us.apache.org/repos/asf/arrow/blob/cf4ef5ef/java/vector/pom.xml
----------------------------------------------------------------------
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index 2605523..a117a2f 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -14,7 +14,7 @@
<parent>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-java-root</artifactId>
- <version>0.4.0</version>
+ <version>0.5.0-SNAPSHOT</version>
</parent>
<artifactId>arrow-vector</artifactId>
<name>Arrow Vectors</name>
[04/14] arrow git commit: ARROW-1034: [PYTHON] Resolve wheel build
issues on Windows
Posted by we...@apache.org.
ARROW-1034: [PYTHON] Resolve wheel build issues on Windows
Author: Max Risuhin <ri...@gmail.com>
Closes #707 from MaxRis/ARROW-1034 and squashes the following commits:
796d1554 [Max Risuhin] ARROW-1034: [PYTHON] Resolve wheel build issues on Windows
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/33117d9a
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/33117d9a
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/33117d9a
Branch: refs/heads/master
Commit: 33117d9a7df9a38fb1100d428852c9f5c9d65e4d
Parents: aa652cb
Author: Max Risuhin <ri...@gmail.com>
Authored: Tue May 23 14:09:36 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Wed May 31 13:45:48 2017 -0400
----------------------------------------------------------------------
ci/msvc-build.bat | 2 +-
python/CMakeLists.txt | 34 +++++++++++++++++-------
python/cmake_modules/FindArrow.cmake | 41 ++++++++++++++++++++++-------
python/cmake_modules/FindParquet.cmake | 41 ++++++++++++++++++++++-------
python/setup.py | 23 +++++++++++-----
5 files changed, 105 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/33117d9a/ci/msvc-build.bat
----------------------------------------------------------------------
diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index f756fc5..d13c11f 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -68,6 +68,6 @@ popd
set PYTHONPATH=
pushd python
-python setup.py build_ext --inplace --with-parquet || exit /B
+python setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp bdist_wheel || exit /B
py.test pyarrow -v -s || exit /B
popd
http://git-wip-us.apache.org/repos/asf/arrow/blob/33117d9a/python/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index 6f48f7f..b02f23b 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -237,7 +237,7 @@ function(bundle_arrow_lib library_path)
configure_file(${LIBRARY_DIR}/${LIBRARY_NAME}.${ARROW_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}.${ARROW_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
COPYONLY)
- else()
+ else(NOT MSVC)
configure_file(${${library_path}}.${ARROW_ABI_VERSION}
${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}.${ARROW_ABI_VERSION}
COPYONLY)
@@ -257,10 +257,17 @@ if (PYARROW_BUNDLE_ARROW_CPP)
bundle_arrow_lib(ARROW_PYTHON_SHARED_LIB)
endif()
-ADD_THIRDPARTY_LIB(arrow
- SHARED_LIB ${ARROW_SHARED_LIB})
-ADD_THIRDPARTY_LIB(arrow_python
- SHARED_LIB ${ARROW_PYTHON_SHARED_LIB})
+if (MSVC)
+ ADD_THIRDPARTY_LIB(arrow
+ SHARED_LIB ${ARROW_SHARED_IMP_LIB})
+ ADD_THIRDPARTY_LIB(arrow_python
+ SHARED_LIB ${ARROW_PYTHON_SHARED_IMP_LIB})
+else()
+ ADD_THIRDPARTY_LIB(arrow
+ SHARED_LIB ${ARROW_SHARED_LIB})
+ ADD_THIRDPARTY_LIB(arrow_python
+ SHARED_LIB ${ARROW_PYTHON_SHARED_LIB})
+endif()
############################################################
# Subdirectories
@@ -312,7 +319,7 @@ if (PYARROW_BUILD_PARQUET)
configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
COPYONLY)
- else()
+ elseif (NOT MSVC)
configure_file(${PARQUET_LIBRARY_DIR}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION}
${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION}
COPYONLY)
@@ -332,10 +339,17 @@ if (PYARROW_BUILD_PARQUET)
#SET(PARQUET_ARROW_SHARED_LIB
# ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX})
endif()
- ADD_THIRDPARTY_LIB(parquet
- SHARED_LIB ${PARQUET_SHARED_LIB})
- ADD_THIRDPARTY_LIB(parquet_arrow
- SHARED_LIB ${PARQUET_ARROW_SHARED_LIB})
+ if (MSVC)
+ ADD_THIRDPARTY_LIB(parquet
+ SHARED_LIB ${PARQUET_SHARED_IMP_LIB})
+ ADD_THIRDPARTY_LIB(parquet_arrow
+ SHARED_LIB ${PARQUET_ARROW_SHARED_IMP_LIB})
+ else()
+ ADD_THIRDPARTY_LIB(parquet
+ SHARED_LIB ${PARQUET_SHARED_LIB})
+ ADD_THIRDPARTY_LIB(parquet_arrow
+ SHARED_LIB ${PARQUET_ARROW_SHARED_LIB})
+ endif()
set(LINK_LIBS
${LINK_LIBS}
parquet_shared
http://git-wip-us.apache.org/repos/asf/arrow/blob/33117d9a/python/cmake_modules/FindArrow.cmake
----------------------------------------------------------------------
diff --git a/python/cmake_modules/FindArrow.cmake b/python/cmake_modules/FindArrow.cmake
index c16a4bf..4c8ed3d 100644
--- a/python/cmake_modules/FindArrow.cmake
+++ b/python/cmake_modules/FindArrow.cmake
@@ -21,6 +21,7 @@
# ARROW_LIBS, directory containing arrow libraries
# ARROW_STATIC_LIB, path to libarrow.a
# ARROW_SHARED_LIB, path to libarrow's shared library
+# ARROW_SHARED_IMP_LIB, path to libarrow's import library (MSVC only)
# ARROW_FOUND, whether arrow has been found
include(FindPkgConfig)
@@ -69,23 +70,45 @@ find_library(ARROW_PYTHON_LIB_PATH NAMES arrow_python
PATHS
${ARROW_SEARCH_LIB_PATH}
NO_DEFAULT_PATH)
+get_filename_component(ARROW_PYTHON_LIBS ${ARROW_PYTHON_LIB_PATH} DIRECTORY)
+
+if (MSVC)
+ SET(CMAKE_FIND_LIBRARY_SUFFIXES ".lib" ".dll")
+
+ if (MSVC AND NOT ARROW_MSVC_STATIC_LIB_SUFFIX)
+ set(ARROW_MSVC_STATIC_LIB_SUFFIX "_static")
+ endif()
+
+ find_library(ARROW_SHARED_LIBRARIES NAMES arrow
+ PATHS ${ARROW_HOME} NO_DEFAULT_PATH
+ PATH_SUFFIXES "bin" )
+
+ find_library(ARROW_PYTHON_SHARED_LIBRARIES NAMES arrow_python
+ PATHS ${ARROW_HOME} NO_DEFAULT_PATH
+ PATH_SUFFIXES "bin" )
+ get_filename_component(ARROW_SHARED_LIBS ${ARROW_SHARED_LIBRARIES} PATH )
+ get_filename_component(ARROW_PYTHON_SHARED_LIBS ${ARROW_PYTHON_SHARED_LIBRARIES} PATH )
+endif ()
if (ARROW_INCLUDE_DIR AND ARROW_LIBS)
set(ARROW_FOUND TRUE)
-
+ set(ARROW_LIB_NAME arrow)
+ set(ARROW_PYTHON_LIB_NAME arrow_python)
if (MSVC)
- set(ARROW_STATIC_LIB ${ARROW_LIB_PATH})
- set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH})
- set(ARROW_SHARED_LIB ${ARROW_STATIC_LIB})
- set(ARROW_PYTHON_SHARED_LIB ${ARROW_PYTHON_STATIC_LIB})
+ set(ARROW_STATIC_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
+ set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIBS}/${ARROW_PYTHON_LIB_NAME}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
+ set(ARROW_SHARED_LIB ${ARROW_SHARED_LIBS}/${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+ set(ARROW_PYTHON_SHARED_LIB ${ARROW_PYTHON_SHARED_LIBS}/${ARROW_PYTHON_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+ set(ARROW_SHARED_IMP_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}.lib)
+ set(ARROW_PYTHON_SHARED_IMP_LIB ${ARROW_PYTHON_LIBS}/${ARROW_PYTHON_LIB_NAME}.lib)
else()
- set(ARROW_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow.a)
- set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_python.a)
+ set(ARROW_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/lib${ARROW_LIB_NAME}.a)
+ set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/lib${ARROW_PYTHON_LIB_NAME}.a)
set(ARROW_JEMALLOC_STATIC_LIB ${ARROW_PYTHON_LIB_PATH}/libarrow_jemalloc.a)
- set(ARROW_SHARED_LIB ${ARROW_LIBS}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX})
+ set(ARROW_SHARED_LIB ${ARROW_LIBS}/lib${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
set(ARROW_JEMALLOC_SHARED_LIB ${ARROW_LIBS}/libarrow_jemalloc${CMAKE_SHARED_LIBRARY_SUFFIX})
- set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/libarrow_python${CMAKE_SHARED_LIBRARY_SUFFIX})
+ set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/lib${ARROW_PYTHON_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
endif()
endif()
http://git-wip-us.apache.org/repos/asf/arrow/blob/33117d9a/python/cmake_modules/FindParquet.cmake
----------------------------------------------------------------------
diff --git a/python/cmake_modules/FindParquet.cmake b/python/cmake_modules/FindParquet.cmake
index de53a29..ef3c645 100644
--- a/python/cmake_modules/FindParquet.cmake
+++ b/python/cmake_modules/FindParquet.cmake
@@ -21,6 +21,7 @@
# PARQUET_LIBS, directory containing parquet libraries
# PARQUET_STATIC_LIB, path to libparquet.a
# PARQUET_SHARED_LIB, path to libparquet's shared library
+# PARQUET_SHARED_IMP_LIB, path to libparquet's import library (MSVC only)
# PARQUET_FOUND, whether parquet has been found
include(FindPkgConfig)
@@ -29,6 +30,24 @@ if(NOT "$ENV{PARQUET_HOME}" STREQUAL "")
set(PARQUET_HOME "$ENV{PARQUET_HOME}")
endif()
+if (MSVC)
+ SET(CMAKE_FIND_LIBRARY_SUFFIXES ".lib" ".dll")
+
+ if (MSVC AND NOT PARQUET_MSVC_STATIC_LIB_SUFFIX)
+ set(PARQUET_MSVC_STATIC_LIB_SUFFIX "_static")
+ endif()
+
+ find_library(PARQUET_SHARED_LIBRARIES NAMES parquet
+ PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
+ PATH_SUFFIXES "bin" )
+
+ find_library(PARQUET_ARROW_SHARED_LIBRARIES NAMES parquet_arrow
+ PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
+ PATH_SUFFIXES "bin" )
+ get_filename_component(PARQUET_SHARED_LIBS ${PARQUET_SHARED_LIBRARIES} PATH )
+ get_filename_component(PARQUET_ARROW_SHARED_LIBS ${PARQUET_ARROW_SHARED_LIBRARIES} PATH )
+endif ()
+
if(PARQUET_HOME)
set(PARQUET_SEARCH_HEADER_PATHS
${PARQUET_HOME}/include
@@ -74,13 +93,14 @@ endif()
if (PARQUET_INCLUDE_DIR AND PARQUET_LIBRARIES)
set(PARQUET_FOUND TRUE)
+ set(PARQUET_LIB_NAME parquet)
if (MSVC)
- set(PARQUET_STATIC_LIB "${PARQUET_LIBRARIES}_static")
- set(PARQUET_SHARED_LIB "${PARQUET_LIBRARIES}")
+ set(PARQUET_STATIC_LIB "${PARQUET_LIBS}/${PARQUET_LIB_NAME}${PARQUET_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}")
+ set(PARQUET_SHARED_LIB "${PARQUET_SHARED_LIBS}/${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}")
+ set(PARQUET_SHARED_IMP_LIB "${PARQUET_LIBS}/${PARQUET_LIB_NAME}.lib")
else()
- set(PARQUET_LIB_NAME libparquet)
- set(PARQUET_STATIC_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}.a)
- set(PARQUET_SHARED_LIB ${PARQUET_LIBS}/${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+ set(PARQUET_STATIC_LIB ${PARQUET_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PARQUET_LIB_NAME}.a)
+ set(PARQUET_SHARED_LIB ${PARQUET_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
endif()
else ()
set(PARQUET_FOUND FALSE)
@@ -89,15 +109,16 @@ endif ()
if (PARQUET_INCLUDE_DIR AND PARQUET_ARROW_LIBRARIES)
set(PARQUET_ARROW_FOUND TRUE)
get_filename_component(PARQUET_ARROW_LIBS ${PARQUET_ARROW_LIBRARIES} PATH)
+ set(PARQUET_ARROW_LIB_NAME parquet_arrow)
if (MSVC)
- set(PARQUET_ARROW_STATIC_LIB "${PARQUET_ARROW_LIBRARIES}_static")
- set(PARQUET_ARROW_SHARED_LIB "${PARQUET_ARROW_LIBRARIES}")
+ set(PARQUET_ARROW_STATIC_LIB "${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}${PARQUET_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}")
+ set(PARQUET_ARROW_SHARED_LIB "${PARQUET_ARROW_SHARED_LIBS}/${PARQUET_ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}")
+ set(PARQUET_ARROW_SHARED_IMP_LIB "${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}.lib")
else()
- set(PARQUET_ARROW_LIB_NAME libparquet_arrow)
set(PARQUET_ARROW_STATIC_LIB
- ${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}.a)
+ ${PARQUET_ARROW_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PARQUET_ARROW_LIB_NAME}.a)
set(PARQUET_ARROW_SHARED_LIB
- ${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+ ${PARQUET_ARROW_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${PARQUET_ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
endif()
else ()
set(PARQUET_ARROW_FOUND FALSE)
http://git-wip-us.apache.org/repos/asf/arrow/blob/33117d9a/python/setup.py
----------------------------------------------------------------------
diff --git a/python/setup.py b/python/setup.py
index 9374af3..7e612c9 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -29,7 +29,7 @@ import Cython
import pkg_resources
-from setuptools import setup, Extension
+from setuptools import setup, Extension, Distribution
from os.path import join as pjoin
@@ -210,6 +210,11 @@ class build_ext(_build_ext):
except OSError:
pass
+ if sys.platform == 'win32':
+ build_prefix = ''
+ else:
+ build_prefix = self.build_type
+
def move_lib(lib_name):
lib_filename = (shared_library_prefix + lib_name +
shared_library_suffix)
@@ -217,14 +222,14 @@ class build_ext(_build_ext):
if sys.platform == 'darwin':
lib_pattern = (shared_library_prefix + lib_name +
".*" + shared_library_suffix[1:])
- libs = glob.glob(pjoin(self.build_type, lib_pattern))
+ libs = glob.glob(pjoin(build_prefix, lib_pattern))
else:
- libs = glob.glob(pjoin(self.build_type, lib_filename) + '*')
+ libs = glob.glob(pjoin(build_prefix, lib_filename) + '*')
# Longest suffix library should be copied, all others symlinked
libs.sort(key=lambda s: -len(s))
print(libs, libs[0])
lib_filename = os.path.basename(libs[0])
- shutil.move(pjoin(self.build_type, lib_filename),
+ shutil.move(pjoin(build_prefix, lib_filename),
pjoin(build_lib, 'pyarrow', lib_filename))
for lib in libs[1:]:
filename = os.path.basename(lib)
@@ -233,10 +238,10 @@ class build_ext(_build_ext):
os.symlink(lib_filename, link_name)
if self.bundle_arrow_cpp:
- print(pjoin(self.build_type, 'include'), pjoin(build_lib, 'pyarrow'))
+ print(pjoin(build_prefix, 'include'), pjoin(build_lib, 'pyarrow'))
if os.path.exists(pjoin(build_lib, 'pyarrow', 'include')):
shutil.rmtree(pjoin(build_lib, 'pyarrow', 'include'))
- shutil.move(pjoin(self.build_type, 'include'), pjoin(build_lib, 'pyarrow'))
+ shutil.move(pjoin(build_prefix, 'include'), pjoin(build_lib, 'pyarrow'))
move_lib("arrow")
move_lib("arrow_python")
if self.with_jemalloc:
@@ -337,11 +342,17 @@ representations of flat and hierarchical data along with multiple
language-bindings for structure manipulation. It also provides IPC
and common algorithm implementations."""
+class BinaryDistribution(Distribution):
+ def has_ext_modules(foo):
+ return True
+
setup(
name="pyarrow",
packages=['pyarrow', 'pyarrow.tests'],
zip_safe=False,
package_data={'pyarrow': ['*.pxd', '*.pyx']},
+ include_package_data=True,
+ distclass=BinaryDistribution,
# Dummy extension to trigger build_ext
ext_modules=[Extension('__dummy__', sources=[])],
[03/14] arrow git commit: ARROW-897: [GLib] Extract CI configuration
for GLib
Posted by we...@apache.org.
ARROW-897: [GLib] Extract CI configuration for GLib
Author: Kouhei Sutou <ko...@clear-code.com>
Closes #713 from kou/glib-travis-split and squashes the following commits:
682f0bd [Kouhei Sutou] [GLib] Build only C++ libraries for GLib CI
7d02eb7 [Kouhei Sutou] [GLib] Extract CI configuration for GLib
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/51b6bf25
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/51b6bf25
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/51b6bf25
Branch: refs/heads/master
Commit: 51b6bf2532646ce1f916613e22272782c918b2ee
Parents: 03e8b54
Author: Kouhei Sutou <ko...@clear-code.com>
Authored: Fri May 26 13:12:44 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Wed May 31 13:45:48 2017 -0400
----------------------------------------------------------------------
.travis.yml | 13 ++++++++++--
ci/travis_before_script_cpp.sh | 42 ++++++++++++++++++++++++++++++-------
ci/travis_env_common.sh | 5 -----
3 files changed, 45 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/51b6bf25/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index b212fcb..36e3c18 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -40,11 +40,9 @@ matrix:
- export CC="gcc-4.9"
- export CXX="g++-4.9"
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
- - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
script:
- $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh
- $TRAVIS_BUILD_DIR/ci/travis_script_python.sh
- - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
- compiler: clang
osx_image: xcode6.4
os: osx
@@ -82,6 +80,17 @@ matrix:
- $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
script:
- $TRAVIS_BUILD_DIR/ci/travis_script_js.sh
+ - compiler: gcc
+ language: cpp
+ os: linux
+ group: deprecated
+ before_script:
+ - export CC="gcc-4.9"
+ - export CXX="g++-4.9"
+ - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library
+ - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh
+ script:
+ - $TRAVIS_BUILD_DIR/ci/travis_script_c_glib.sh
before_install:
- ulimit -c unlimited -S
http://git-wip-us.apache.org/repos/asf/arrow/blob/51b6bf25/ci/travis_before_script_cpp.sh
----------------------------------------------------------------------
diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
index 7d4ecb7..9908735 100755
--- a/ci/travis_before_script_cpp.sh
+++ b/ci/travis_before_script_cpp.sh
@@ -15,11 +15,24 @@
set -ex
+if [ "$1" == "--only-library" ]; then
+ only_library_mode=yes
+else
+ only_library_mode=no
+fi
+
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
-source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
-# Set up C++ toolchain from conda-forge packages for faster builds
-conda create -y -q -p $CPP_TOOLCHAIN python=2.7 flatbuffers rapidjson
+if [ $only_library_mode == "no" ]; then
+ # C++ toolchain
+ export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain
+ export FLATBUFFERS_HOME=$CPP_TOOLCHAIN
+ export RAPIDJSON_HOME=$CPP_TOOLCHAIN
+
+ # Set up C++ toolchain from conda-forge packages for faster builds
+ source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
+ conda create -y -q -p $CPP_TOOLCHAIN python=2.7 flatbuffers rapidjson
+fi
if [ $TRAVIS_OS_NAME == "osx" ]; then
brew update > /dev/null
@@ -32,18 +45,31 @@ pushd $ARROW_CPP_BUILD_DIR
CMAKE_COMMON_FLAGS="\
-DARROW_BUILD_BENCHMARKS=ON \
--DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL"
+-DCMAKE_INSTALL_PREFIX=$ARROW_CPP_INSTALL
+-DARROW_NO_DEPRECATED_API=ON"
+CMAKE_LINUX_FLAGS=""
+CMAKE_OSX_FLAGS=""
+
+if [ $only_library_mode == "yes" ]; then
+ CMAKE_COMMON_FLAGS="\
+$CMAKE_COMMON_FLAGS \
+-DARROW_BUILD_TESTS=OFF \
+-DARROW_BUILD_UTILITIES=OFF"
+else
+ CMAKE_LINUX_FLAGS="\
+$CMAKE_LINUX_FLAGS \
+-DARROW_TEST_MEMCHECK=ON"
+fi
if [ $TRAVIS_OS_NAME == "linux" ]; then
- cmake -DARROW_TEST_MEMCHECK=on \
- $CMAKE_COMMON_FLAGS \
+ cmake $CMAKE_COMMON_FLAGS \
+ $CMAKE_LINUX_FLAGS \
-DARROW_CXXFLAGS="-Wconversion -Werror" \
- -DARROW_NO_DEPRECATED_API=on \
$ARROW_CPP_DIR
else
cmake $CMAKE_COMMON_FLAGS \
+ $CMAKE_OSX_FLAGS \
-DARROW_CXXFLAGS=-Werror \
- -DARROW_NO_DEPRECATED_API=on \
$ARROW_CPP_DIR
fi
http://git-wip-us.apache.org/repos/asf/arrow/blob/51b6bf25/ci/travis_env_common.sh
----------------------------------------------------------------------
diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh
index b19fafa..cef2d65 100755
--- a/ci/travis_env_common.sh
+++ b/ci/travis_env_common.sh
@@ -16,11 +16,6 @@ export MINICONDA=$HOME/miniconda
export PATH="$MINICONDA/bin:$PATH"
export CONDA_PKGS_DIRS=$HOME/.conda_packages
-# C++ toolchain
-export CPP_TOOLCHAIN=$TRAVIS_BUILD_DIR/cpp-toolchain
-export FLATBUFFERS_HOME=$CPP_TOOLCHAIN
-export RAPIDJSON_HOME=$CPP_TOOLCHAIN
-
export ARROW_CPP_DIR=$TRAVIS_BUILD_DIR/cpp
export ARROW_PYTHON_DIR=$TRAVIS_BUILD_DIR/python
export ARROW_C_GLIB_DIR=$TRAVIS_BUILD_DIR/c_glib
[13/14] arrow git commit: ARROW-1069: Add instructions for publishing
maven artifacts
Posted by we...@apache.org.
ARROW-1069: Add instructions for publishing maven artifacts
Author: Julien Le Dem <ju...@apache.org>
Closes #716 from julienledem/ARROW-1069 and squashes the following commits:
8896189 [Julien Le Dem] ARROW-1069: Add instructions for publishing maven artifacts
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/03e8b54d
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/03e8b54d
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/03e8b54d
Branch: refs/heads/master
Commit: 03e8b54d8242daf0358e39cd2ccb1f0599223c9e
Parents: 8a700cc
Author: Julien Le Dem <ju...@apache.org>
Authored: Fri May 26 12:47:14 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Wed May 31 13:45:48 2017 -0400
----------------------------------------------------------------------
dev/release/README | 21 ++++++++++++++++++++-
1 file changed, 20 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/03e8b54d/dev/release/README
----------------------------------------------------------------------
diff --git a/dev/release/README b/dev/release/README
index cf68028..c53d1d8 100644
--- a/dev/release/README
+++ b/dev/release/README
@@ -15,10 +15,11 @@ source dev/release/setup-gpg-agent.sh
sh dev/release/00-prepare.sh 0.1.0 0.1.1
# push the tag
git push apache apache-arrow-0.1.0
-# tag and push to maven repo (repo will have to be finalized separately)
+# tag and stage artifacts to maven repo (repo will have to be finalized separately)
sh dev/release/01-perform.sh
# create the source release
sh dev/release/02-source.sh 0.1.0 0
+# once the vote has passed, publish the staged maven artifacts (see bellow)
useful commands:
- to set the mvn version in the poms
@@ -30,3 +31,21 @@ eval $(gpg-agent --daemon --allow-preset-passphrase)
gpg --use-agent -s LICENSE.txt
- delete tag localy
git tag -d apache-arrow-0.1.0
+
+How to stage maven artifacts:
+artifacts get staged during the perform phase of the scripts above.
+If you need to stage the artifacts again follow the instructions bellow:
+# checkout the relese tag
+git checkout apache-arrow-0.1.0
+# setup the gpg agent for signing artifacts
+source dev/release/setup-gpg-agent.sh
+# go in the java subfolder
+cd java
+# stage the artifacts
+mvn -Papache-release deploy
+
+How to publish the staged artifacts:
+Logon to the apache repository: https://repository.apache.org/#stagingRepositories
+Select the arrow staging repository you just just created: orgapachearrow-100x
+Click the "close" button
+Once validation has passed, click the "release" button